Dump additional apis (#3684)

* ✨ Allow JSON generation for all API

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 👷 Export AS API JSON

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* ✨ Handle missing security definitions

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 👷 Export Push Gateway API JSON

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 🎨 Improve identation

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 🐛 Fix successive reference handling

Fixes #3689

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 👷 Export Server-Server API JSON

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 📝 Remove obsolete comment

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* 🐛 Make properties objects, as they MUST be

Signed-off-by: Alexandre Franke <alexandre.franke@matrix.org>

* Revert "👷 Export Server-Server API JSON"

This reverts commit 061f91c2cbcb6dff35c74177226da106826b4214.

Co-authored-by: Alexandre Franke <alexandre.franke@matrix.org>

.github/workflows/main.yml

@@ -26,7 +26,9 @@ jobs:
       - name: "🔎 Run validator"
         working-directory: "./scripts"
         run: |
+          node validator.js -s "../data/api/application-service"
           node validator.js -s "../data/api/client-server"
+          node validator.js -s "../data/api/push-gateway"
 
   check-examples:
     name: "🔎 Check Event schema examples"
@@ -76,7 +78,16 @@ jobs:
           # The output path matches the final deployment path at spec.matrix.org
           scripts/dump-swagger.py \
             --base-url "https://spec.matrix.org${{ needs.calculate-baseurl.outputs.baseURL }}" \
+            --api application-service \
+            -o spec/application-service-api/api.json
+          scripts/dump-swagger.py \
+            --base-url "https://spec.matrix.org${{ needs.calculate-baseurl.outputs.baseURL }}" \
+            --api client-server \
             -o spec/client-server-api/api.json
+          scripts/dump-swagger.py \
+            --base-url "https://spec.matrix.org${{ needs.calculate-baseurl.outputs.baseURL }}" \
+            --api push-gateway \
+            -o spec/push-gateway-api/api.json
           tar -czf openapi.tar.gz spec
       - name: "📤 Artifact upload"
         uses: actions/upload-artifact@v2

data/api/server-server/definitions/send_join_response.yaml

@@ -34,7 +34,7 @@ properties:
         on the room version - check the [room version specification](/rooms) for precise event formats.
       schema:
         type: object
-        properties: []
+        properties: {}
       example:
         $ref: "../examples/minimal_pdu.json"
   state:
@@ -52,7 +52,7 @@ properties:
         on the room version - check the [room version specification](/rooms) for precise event formats.
       schema:
         type: object
-        properties: []
+        properties: {}
       example:
         $ref: "../examples/minimal_pdu.json"
   event:

data/api/server-server/definitions/single_pdu_transaction.yaml

@@ -26,7 +26,7 @@ properties:
       description: |-
         The [PDUs](/server-server-api/#pdus) contained in the transaction. The event format varies depending
         on the room version - check the [room version specification](/rooms) for precise event formats.
-      properties: []
+      properties: {}
       example:
         $ref: "../examples/minimal_pdu.json"
 required: ['origin', 'origin_server_ts', 'pdus']

data/api/server-server/definitions/transaction.yaml

@@ -41,7 +41,7 @@ properties:
       description: |-
         The [PDUs](/server-server-api/#pdus) contained in the transaction. The event format varies depending
         on the room version - check the [room version specification](/rooms) for precise event formats.
-      properties: []
+      properties: {}
       example:
         $ref: "../examples/minimal_pdu.json"
 required: ['origin', 'origin_server_ts', 'pdus']

data/api/server-server/definitions/unlimited_pdu_transaction.yaml

@@ -27,7 +27,7 @@ properties:
       description: |-
         The [PDUs](/server-server-api/#pdus) contained in the transaction. The event format varies depending
         on the room version - check the [room version specification](/rooms) for precise event formats.
-      properties: []
+      properties: {}
       example:
         $ref: "../examples/minimal_pdu.json"
 required: ['origin', 'origin_server_ts', 'pdus']

data/api/server-server/event_auth.yaml

@@ -64,7 +64,7 @@ paths:
                 items:
                   type: object
                   title: PDU
-                  properties: []
+                  properties: {}
                   example:
                     $ref: "examples/minimal_pdu.json"
             required: ['auth_chain']

data/api/server-server/events.yaml

@@ -69,7 +69,7 @@ paths:
                     The [PDUs](/server-server-api/#pdus) contained in the auth chain. The event format
                     varies depending on the room version - check the [room version specification](/rooms)
                     for precise event formats.
-                  properties: []
+                  properties: {}
                   example:
                     $ref: "examples/minimal_pdu.json"
               pdus:
@@ -85,7 +85,7 @@ paths:
                     The [PDUs](/server-server-api/#pdus) for the fully resolved state of the room. The event format
                     varies depending on the room version - check the [room version specification](/rooms)
                     for precise event formats.
-                  properties: []
+                  properties: {}
                   example:
                     $ref: "examples/minimal_pdu.json"
             required: ['auth_chain', 'pdus']

data/api/server-server/transactions.yaml

@@ -69,7 +69,7 @@ paths:
                       $ref: "definitions/edu.yaml"
           example: {
             "$ref": "examples/transaction.json",
-            "edus": [{"$ref": "edu.json"}] # Relative to the examples directory
+            "edus": [{"$ref": "examples/edu.json"}]
           }
       responses:
         200:

scripts/dump-swagger.py

@@ -37,11 +37,18 @@ def resolve_references(path, schema):
         # do $ref first
         if '$ref' in schema:
             value = schema['$ref']
+            previous_path = path
             path = os.path.join(os.path.dirname(path), value)
+            try:
                 with open(path, encoding="utf-8") as f:
                     ref = yaml.safe_load(f)
                 result = resolve_references(path, ref)
                 del schema['$ref']
+                path = previous_path
+            except FileNotFoundError:
+                print("Resolving {}".format(schema))
+                print("File not found: {}".format(path))
+                result = {}
         else:
             result = {}
 
@@ -87,11 +94,24 @@ parser.add_argument(
     %(default)s""",
 )
 parser.add_argument(
-    "--client_release", "-c", metavar="LABEL",
+    "--spec-release", "-r", metavar="LABEL",
     default="unstable",
-    help="""The client-server release version to generate for. Default:
+    help="""The spec release version to generate for. Default:
     %(default)s""",
 )
+available_apis = {
+    "client-server": "Matrix Client-Server API",
+    "server-server": "Matrix Server-Server API",
+    "application-service": "Matrix Application Service API",
+    "identity": "Matrix Identity Service API",
+    "push-gateway": "Matrix Push Gateway API",
+}
+parser.add_argument(
+    "--api",
+    default="client-server",
+    choices=available_apis,
+    help="""The API to generate for. Default: %(default)s""",
+)
 parser.add_argument(
     "-o", "--output",
     default=os.path.join(scripts_dir, "swagger", "api-docs.json"),
@@ -100,7 +120,8 @@ parser.add_argument(
 args = parser.parse_args()
 
 output_file = os.path.abspath(args.output)
-release_label = args.client_release
+release_label = args.spec_release
+selected_api = args.api
 
 major_version = release_label
 match = re.match("^(r\d+)(\.\d+)*$", major_version)
@@ -119,18 +140,20 @@ output = {
     # The servers value will be picked up by RapiDoc to provide a way
     # to switch API servers. Useful when one wants to test compliance
     # of their server with the API.
-    "servers": [{
+    "servers": [
+        {
             "url": "https://{homeserver_address}/",
             "variables": {
                 "homeserver_address": {
                     "default": "matrix-client.matrix.org",
                     "description": "The base URL for your homeserver",
                 }
+            },
         }
-    }],
+    ],
     "schemes": ["https"],
     "info": {
-        "title": "Matrix Client-Server API",
+        "title": available_apis[selected_api],
         "version": release_label,
     },
     "securityDefinitions": {},
@@ -138,15 +161,17 @@ output = {
     "swagger": "2.0",
 }
 
-cs_api_dir = os.path.join(api_dir, 'client-server')
+selected_api_dir = os.path.join(api_dir, selected_api)
-with open(os.path.join(cs_api_dir, 'definitions',
+try:
-                       'security.yaml')) as f:
+    with open(os.path.join(selected_api_dir, 'definitions', 'security.yaml')) as f:
         output['securityDefinitions'] = yaml.safe_load(f)
+except FileNotFoundError:
+    print("No security definitions available for this API")
 
-for filename in os.listdir(cs_api_dir):
+for filename in os.listdir(selected_api_dir):
     if not filename.endswith(".yaml"):
         continue
-    filepath = os.path.join(cs_api_dir, filename)
+    filepath = os.path.join(selected_api_dir, filename)
 
     print("Reading swagger API: %s" % filepath)
     with open(filepath, "r") as f: