Convert build scripts to python 3

.gitignore

@@ -1,7 +1,7 @@
 /api/node_modules
 /assets
 /assets.tar.gz
-/env
+/env*
 /scripts/gen
 /scripts/continuserv/continuserv
 /scripts/speculator/speculator

README.rst

@@ -41,7 +41,7 @@ specs and event schemas in this repository.
 Preparation
 -----------
 
-To use the scripts, it is best to create a Python 2.x virtualenv as follows::
+To use the scripts, it is best to create a Python 3.5+ virtualenv as follows::
 
   virtualenv env
   env/bin/pip install -r scripts/requirements.txt

api/check_examples.py

@@ -88,9 +88,9 @@ def check_swagger_file(filepath):
     with open(filepath) as f:
         swagger = yaml.load(f)
 
-    for path, path_api in swagger.get('paths', {}).items():
+    for path, path_api in list(swagger.get('paths', {}).items()):
 
-        for method, request_api in path_api.items():
+        for method, request_api in list(path_api.items()):
             request = "%s %s" % (method.upper(), path)
             for parameter in request_api.get('parameters', ()):
                 if parameter['in'] == 'body':
@@ -100,7 +100,7 @@ def check_swagger_file(filepath):
                 responses = request_api['responses']
             except KeyError:
                 raise ValueError("No responses for %r" % (request,))
-            for code, response in responses.items():
+            for code, response in list(responses.items()):
                 check_response(filepath, request, code, response)
 
 

scripts/dump-swagger.py

@@ -94,16 +94,16 @@ for filename in os.listdir(cs_api_dir):
         api = units.resolve_references(filepath, api)
 
         basePath = api['basePath']
-        for path, methods in api["paths"].items():
+        for path, methods in list(api["paths"].items()):
             path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%',
                                              major_version)
-            for method, spec in methods.items():
+            for method, spec in list(methods.items()):
-                if "tags" in spec.keys():
+                if "tags" in list(spec.keys()):
                     if path not in output["paths"]:
                         output["paths"][path] = {}
                     output["paths"][path][method] = spec
 
-print "Generating %s" % output_file
+print("Generating %s" % output_file)
 
 try:
     os.makedirs(os.path.dirname(output_file))

scripts/gendoc.py

@@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__))
 docs_dir = os.path.dirname(script_dir)
 spec_dir = os.path.join(docs_dir, "specification")
 tmp_dir = os.path.join(script_dir, "tmp")
+changelog_dir = os.path.join(docs_dir, "changelogs")
 
 VERBOSE = False
 
@@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles):
 
 def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
     # string are file paths to RST blobs
-    if isinstance(file_info, basestring):
+    if isinstance(file_info, str):
         log("%s %s" % (">" * (1 + title_level), file_info))
         with open(os.path.join(spec_dir, file_info), "r") as f:
             rst = None
@@ -194,7 +195,7 @@ def build_spec(target, out_filename):
                 spec_dir=spec_dir,
                 adjust_titles=True
             )
-            outfile.write(section)
+            outfile.write(section.encode('UTF-8'))
 
 
 """
@@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets):
 def addAnchors(path):
     log("add anchors %s" % path)
 
-    with open(path, "r") as f:
+    with open(path, "rb") as f:
         lines = f.readlines()
 
     replacement = replacement = r'<p><a class="anchor" id="\3"></a></p>\n\1'
-    with open(path, "w") as f:
+    with open(path, "wb") as f:
         for line in lines:
+            line = line.decode("UTF-8")
             line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
             line = re.sub(r'(<div class="section" (id)="(.*?)">)', replacement, line.rstrip())
-            f.write(line + "\n")
+            f.write((line + "\n").encode('UTF-8'))
 
 
 def run_through_template(input_files, set_verbose, substitutions):
@@ -297,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions):
         "-i", "matrix_templates",
     ]
 
-    for k, v in substitutions.items():
+    for k, v in list(substitutions.items()):
         args.append("--substitution=%s=%s" % (k, v))
 
     if set_verbose:
@@ -357,14 +359,14 @@ def get_build_target(all_targets, target_name):
         for i, entry in enumerate(group):
             if isinstance(entry, dict):
                 group[i] = {
-                    (rel_depth + depth): v for (rel_depth, v) in entry.items()
+                    (rel_depth + depth): v for (rel_depth, v) in list(entry.items())
                 }
         return group
 
     resolved_files = []
     for file_entry in target["files"]:
         # file_entry is a group id
-        if isinstance(file_entry, basestring) and file_entry.startswith("group:"):
+        if isinstance(file_entry, str) and file_entry.startswith("group:"):
             group = get_group(file_entry, 0)
             # The group may be resolved to a list of file entries, in which case
             # we want to extend the array to insert each of them rather than
@@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name):
         # file_entry is a dict which has more file entries as values
         elif isinstance(file_entry, dict):
             resolved_entry = {}
-            for (depth, entry) in file_entry.iteritems():
+            for (depth, entry) in list(file_entry.items()):
-                if not isinstance(entry, basestring):
+                if not isinstance(entry, str):
                     raise Exception(
                         "Double-nested depths are not supported. Entry: %s" % (file_entry,)
                     )
@@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name):
     return build_target
 
 def log(line):
-    print "gendoc: %s" % line
+    print("gendoc: %s" % line)
 
 def logv(line):
     if VERBOSE:
-        print "gendoc:V: %s" % line
+        print("gendoc:V: %s" % line)
 
 
 def cleanup_env():
@@ -427,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions):
         target_defs = yaml.load(targ_file.read())
 
     if targets == ["all"]:
-        targets = target_defs["targets"].keys()
+        targets = list(target_defs["targets"].keys())
 
     log("Building spec [targets=%s]" % targets)
 
@@ -441,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions):
         templated_files[target_name] = templated_file
 
     # we do all the templating at once, because it's slow
-    run_through_template(templated_files.values(), VERBOSE, substitutions)
+    run_through_template(list(templated_files.values()), VERBOSE, substitutions)
 
     stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css"))
 
-    for target_name, templated_file in templated_files.iteritems():
+    for target_name, templated_file in list(templated_files.items()):
         target = target_defs["targets"].get(target_name)
         version_label = None
         if target:
             version_label = target.get("version_label")
             if version_label:
-                for old, new in substitutions.items():
+                for old, new in list(substitutions.items()):
                     version_label = version_label.replace(old, new)
 
         rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,))
@@ -479,8 +481,8 @@ def main(targets, dest_dir, keep_intermediates, substitutions):
 def list_targets():
     with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file:
         target_defs = yaml.load(targ_file.read())
-    targets = target_defs["targets"].keys()
+    targets = list(target_defs["targets"].keys())
-    print "\n".join(targets)
+    print("\n".join(targets))
 
 
 def extract_major(s):

scripts/proposals.py

@@ -14,7 +14,7 @@ prs = set()
 def getpage(url, page):
     resp = requests.get(url + str(page))
 
-    for link in resp.links.values():
+    for link in list(resp.links.values()):
         if link['rel'] == 'last':
             pagecount = re.search('page=(.+?)', link['url']).group(1)
 

scripts/swagger-http-server.py

@@ -19,14 +19,14 @@
 
 import argparse
 import os
-import SimpleHTTPServer
+import http.server
-import SocketServer
+import socketserver
 
 # Thanks to http://stackoverflow.com/a/13354482
-class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
     def end_headers(self):
         self.send_my_headers()
-        SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self)
+        http.server.SimpleHTTPRequestHandler.end_headers(self)
 
     def send_my_headers(self):
         self.send_header("Access-Control-Allow-Origin", "*")
@@ -49,7 +49,7 @@ if __name__ == '__main__':
 
     os.chdir(args.swagger_dir)
 
-    httpd = SocketServer.TCPServer(("localhost", args.port),
+    httpd = socketserver.TCPServer(("localhost", args.port),
                                    MyHTTPRequestHandler)
-    print "Serving at http://localhost:%i/api-docs.json" % args.port
+    print("Serving at http://localhost:%i/api-docs.json" % args.port)
     httpd.serve_forever()

scripts/templating/batesian/__init__.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from sets import Set
 
 
 class AccessKeyStore(object):
@@ -22,10 +21,10 @@ class AccessKeyStore(object):
         if not existing_data:
             existing_data = {}
         self.data = existing_data
-        self.accessed_set = Set()
+        self.accessed_set = set()
 
     def keys(self):
-        return self.data.keys()
+        return list(self.data.keys())
 
     def add(self, key, unit_dict):
         self.data[key] = unit_dict
@@ -35,5 +34,5 @@ class AccessKeyStore(object):
         return self.data[key]
 
     def get_unaccessed_set(self):
-        data_list = Set(self.data.keys())
+        data_list = set(self.data.keys())
         return data_list - self.accessed_set

scripts/templating/batesian/sections.py

@@ -29,7 +29,7 @@ class Sections(object):
 
     def log(self, text):
         if self.debug:
-            print "batesian:sections: %s" % text
+            print("batesian:sections: %s" % text)
 
     def get_sections(self):
         render_list = inspect.getmembers(self, predicate=inspect.ismethod)
@@ -40,7 +40,7 @@ class Sections(object):
             section_key = func_name[len("render_"):]
             self.log("Generating section '%s'" % section_key)
             section = func()
-            if isinstance(section, basestring):
+            if isinstance(section, str):
                 if section_key in section_dict:
                     raise Exception(
                         ("%s : Section %s already exists. It must have been " +
@@ -54,8 +54,8 @@ class Sections(object):
                 )
             elif isinstance(section, dict):
                 self.log("  Generated multiple sections:")
-                for (k, v) in section.iteritems():
+                for (k, v) in list(section.items()):
-                    if not isinstance(k, basestring) or not isinstance(v, basestring):
+                    if not isinstance(k, str) or not isinstance(v, str):
                         raise Exception(
                             ("Method %s returned multiple sections as a dict but " +
                             "expected the dict elements to be strings but they aren't.") %

scripts/templating/batesian/units.py

@@ -41,7 +41,7 @@ class Units(object):
             trace = inspect.stack()
             if len(trace) > 1 and len(trace[1]) > 2:
                 func_name = trace[1][3] + ":"
-            print "batesian:units:%s %s" % (func_name, text)
+            print("batesian:units:%s %s" % (func_name, text))
 
     def get_units(self, debug=False):
         unit_list = inspect.getmembers(self, predicate=inspect.ismethod)
@@ -50,7 +50,7 @@ class Units(object):
             if not func_name.startswith("load_"):
                 continue
             unit_key = func_name[len("load_"):]
-            if len(inspect.getargs(func.func_code).args) > 1:
+            if len(inspect.getargs(func.__code__).args) > 1:
                 unit_dict[unit_key] = func(self.substitutions)
             else:
                 unit_dict[unit_key] = func()

scripts/templating/build.py

@@ -63,6 +63,7 @@ import sys
 from textwrap import TextWrapper
 
 from matrix_templates.units import TypeTableRow
+from functools import reduce
 
 
 def create_from_template(template, sections):
@@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
             return reduce(max, rowwidths,
                           default if default is not None else default_width)
 
-        results = map(colwidth, keys, defaults)
+        results = list(map(colwidth, keys, defaults))
         return results
 
     # make Jinja aware of the templates and filters
@@ -167,16 +168,16 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
 
     # print out valid section keys if no file supplied
     if not files:
-        print "\nValid template variables:"
+        print("\nValid template variables:")
-        for key in sections.keys():
+        for key in list(sections.keys()):
             sec_text = "" if (len(sections[key]) > 75) else (
                 "(Value: '%s')" % sections[key]
             )
             sec_info = "%s characters" % len(sections[key])
             if sections[key].count("\n") > 0:
                 sec_info += ", %s lines" % sections[key].count("\n")
-            print "  %s" % key
+            print("  %s" % key)
-            print "      %s %s" % (sec_info, sec_text)
+            print("      %s %s" % (sec_info, sec_text))
         return
 
     # check the input files and substitute in sections where required
@@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
 def process_file(env, sections, filename, output_filename):
     log("Parsing input template: %s" % filename)
 
-    with open(filename, "r") as file_stream:
+    with open(filename, "rb") as file_stream:
-        temp_str = file_stream.read().decode("utf-8")
+        temp_str = file_stream.read().decode('UTF-8')
 
     # do sanity checking on the template to make sure they aren't reffing things
     # which will never be replaced with a section.
@@ -210,16 +211,16 @@ def process_file(env, sections, filename, output_filename):
     # Do these substitutions outside of the ordinary templating system because
     # we want them to apply to things like the underlying swagger used to
     # generate the templates, not just the top-level sections.
-    for old, new in substitutions.items():
+    for old, new in list(substitutions.items()):
         output = output.replace(old, new)
 
-    with open(output_filename, "w") as f:
+    with open(output_filename, "wb") as f:
-        f.write(output.encode("utf-8"))
+        f.write(output.encode('UTF-8'))
     log("Output file for: %s" % output_filename)
 
 
 def log(line):
-    print "batesian: %s" % line
+    print("batesian: %s" % line)
 
 if __name__ == '__main__':
     parser = ArgumentParser(

scripts/templating/matrix_templates/__init__.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from sections import MatrixSections
+from .sections import MatrixSections
-from units import MatrixUnits
+from .units import MatrixUnits
 import os
 
 exports = {

scripts/templating/matrix_templates/sections.py

@@ -86,7 +86,7 @@ class MatrixSections(Sections):
     # the key is the section name and the value is the value of the section
     def render_group_http_apis(self):
         # map all swagger_apis to the form $GROUP_http_api
-        swagger_groups = self.units.get("swagger_apis").keys()
+        swagger_groups = list(self.units.get("swagger_apis").keys())
         renders = {}
         for group in swagger_groups:
             sortFnOrPathList = None
@@ -134,7 +134,7 @@ class MatrixSections(Sections):
             "m.room.message#m.file"
         ]
         other_msgtypes = [
-            k for k in schemas.keys() if k.startswith("m.room.message#") and
+            k for k in list(schemas.keys()) if k.startswith("m.room.message#") and
             k not in msgtype_order
         ]
         for event_name in (msgtype_order + other_msgtypes):

scripts/templating/matrix_templates/units.py

@@ -29,11 +29,12 @@ import os.path
 import re
 import subprocess
 import sys
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import yaml
+from functools import reduce
 
 matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc),
-                      range(1, 5), os.path.abspath(__file__))
+                      list(range(1, 5)), os.path.abspath(__file__))
 
 HTTP_APIS = {
     os.path.join(matrix_doc_dir, "api/application-service"): "as",
@@ -124,7 +125,7 @@ def resolve_references(path, schema):
         else:
             result = OrderedDict()
 
-        for key, value in schema.items():
+        for key, value in list(schema.items()):
             result[key] = resolve_references(path, value)
         return result
     elif isinstance(schema, list):
@@ -147,7 +148,7 @@ def inherit_parents(obj):
     # settings defined in the child take priority over the parents, so we
     # iterate through the parents first, and then overwrite with the settings
     # from the child.
-    for p in map(inherit_parents, parents) + [obj]:
+    for p in list(map(inherit_parents, parents)) + [obj]:
         # child blats out type, title and description
         for key in ('type', 'title', 'description'):
             if p.get(key):
@@ -209,7 +210,7 @@ def get_json_schema_object_fields(obj, enforce_title=False):
         props = obj.get("patternProperties")
         if props:
             # try to replace horrible regex key names with pretty x-pattern ones
-            for key_name in props.keys():
+            for key_name in list(props.keys()):
                 pretty_key = props[key_name].get("x-pattern")
                 if pretty_key:
                     props[pretty_key] = props[key_name]
@@ -250,12 +251,12 @@ def get_json_schema_object_fields(obj, enforce_title=False):
             tables.extend(res["tables"])
             logger.debug("Done property %s" % key_name)
 
-        except Exception, e:
+        except Exception as e:
             e2 = Exception("Error reading property %s.%s: %s" %
                            (obj_title, key_name, str(e)))
             # throw the new exception with the old stack trace, so that
             # we don't lose information about where the error occurred.
-            raise e2, None, sys.exc_info()[2]
+            raise e2.with_traceback(sys.exc_info()[2])
 
     tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows))
 
@@ -380,7 +381,7 @@ def get_example_for_schema(schema):
         if 'properties' not in schema:
             raise Exception('"object" property has neither properties nor example')
         res = OrderedDict()
-        for prop_name, prop in schema['properties'].iteritems():
+        for prop_name, prop in list(schema['properties'].items()):
             logger.debug("Parsing property %r" % prop_name)
             prop_example = get_example_for_schema(prop)
             res[prop_name] = prop_example
@@ -523,7 +524,7 @@ class MatrixUnits(Units):
 
                 if param_loc == "path":
                     path_template = path_template.replace(
-                        "{%s}" % param_name, urllib.quote(example)
+                        "{%s}" % param_name, urllib.parse.quote(example)
                     )
                 elif param_loc == "query":
                     if type(example) == list:
@@ -532,7 +533,7 @@ class MatrixUnits(Units):
                     else:
                         example_query_params.append((param_name, example))
 
-            except Exception, e:
+            except Exception as e:
                 raise Exception("Error handling parameter %s" % param_name, e)
         # endfor[param]
         good_response = None
@@ -556,14 +557,14 @@ class MatrixUnits(Units):
                 )
             if "headers" in good_response:
                 headers = TypeTable()
-                for (header_name, header) in good_response["headers"].iteritems():
+                for (header_name, header) in list(good_response["headers"].items()):
                     headers.add_row(
                         TypeTableRow(key=header_name, title=header["type"],
                                      desc=header["description"]),
                     )
                 endpoint["res_headers"] = headers
         query_string = "" if len(
-            example_query_params) == 0 else "?" + urllib.urlencode(
+            example_query_params) == 0 else "?" + urllib.parse.urlencode(
             example_query_params)
         if example_body:
             endpoint["example"][
@@ -605,17 +606,17 @@ class MatrixUnits(Units):
             body_tables = req_body_tables[1:]
             endpoint_data['req_body_tables'].extend(body_tables)
 
-        except Exception, e:
+        except Exception as e:
             e2 = Exception(
                 "Error decoding body of API endpoint %s %s: %s" %
                 (endpoint_data["method"], endpoint_data["path"], e)
             )
-            raise e2, None, sys.exc_info()[2]
+            raise e2.with_traceback(sys.exc_info()[2])
 
 
     def load_swagger_apis(self):
         apis = {}
-        for path, suffix in HTTP_APIS.items():
+        for path, suffix in list(HTTP_APIS.items()):
             for filename in os.listdir(path):
                 if not filename.endswith(".yaml"):
                     continue
@@ -711,12 +712,12 @@ class MatrixUnits(Units):
                     if filename != event_name:
                         examples[event_name] = examples.get(event_name, [])
                         examples[event_name].append(example)
-            except Exception, e:
+            except Exception as e:
                 e2 = Exception("Error reading event example "+filepath+": "+
                                str(e))
                 # throw the new exception with the old stack trace, so that
                 # we don't lose information about where the error occurred.
-                raise e2, None, sys.exc_info()[2]
+                raise e2.with_traceback(sys.exc_info()[2])
 
         return examples
 
@@ -730,12 +731,12 @@ class MatrixUnits(Units):
             filepath = os.path.join(path, filename)
             try:
                 schemata[filename] = self.read_event_schema(filepath)
-            except Exception, e:
+            except Exception as e:
                 e2 = Exception("Error reading event schema "+filepath+": "+
                                str(e))
                 # throw the new exception with the old stack trace, so that
                 # we don't lose information about where the error occurred.
-                raise e2, None, sys.exc_info()[2]
+                raise e2.with_traceback(sys.exc_info()[2])
 
         return schemata
 
@@ -871,7 +872,7 @@ class MatrixUnits(Units):
                 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('UTF-8')
         except subprocess.CalledProcessError:
             git_branch = ""
         try:
@@ -879,7 +880,7 @@ class MatrixUnits(Units):
                 ['git', 'describe', '--exact-match'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('UTF-8')
             git_tag = "tag=" + git_tag
         except subprocess.CalledProcessError:
             git_tag = ""
@@ -888,7 +889,7 @@ class MatrixUnits(Units):
                 ['git', 'rev-parse', '--short', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('UTF-8')
         except subprocess.CalledProcessError:
             git_commit = ""
         try:
@@ -897,7 +898,7 @@ class MatrixUnits(Units):
                 ['git', 'describe', '--dirty=' + dirty_string, "--all"],
                 stderr=null,
                 cwd=cwd,
-            ).strip().endswith(dirty_string)
+            ).strip().decode('UTF-8').endswith(dirty_string)
             git_dirty = "dirty" if is_dirty else ""
         except subprocess.CalledProcessError:
             git_dirty = ""
@@ -908,7 +909,7 @@ class MatrixUnits(Units):
                 s for s in
                 (git_branch, git_tag, git_commit, git_dirty,)
                 if s
-            ).encode("ascii")
+            ).encode("ascii").decode('ascii')
         return {
             "string": git_version,
             "revision": git_commit