From f54d5a40398c3c6a1c252b01d38e50a850685dee Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 15:19:14 -0600 Subject: [PATCH 01/24] Convert build scripts to python 3 --- .gitignore | 2 +- README.rst | 2 +- api/check_examples.py | 6 +-- scripts/dump-swagger.py | 8 +-- scripts/gendoc.py | 38 +++++++------- scripts/proposals.py | 2 +- scripts/swagger-http-server.py | 12 ++--- scripts/templating/batesian/__init__.py | 7 ++- scripts/templating/batesian/sections.py | 8 +-- scripts/templating/batesian/units.py | 4 +- scripts/templating/build.py | 23 ++++----- .../templating/matrix_templates/__init__.py | 4 +- .../templating/matrix_templates/sections.py | 4 +- scripts/templating/matrix_templates/units.py | 49 ++++++++++--------- 14 files changed, 86 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index 84ac4951..58b6099e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /api/node_modules /assets /assets.tar.gz -/env +/env* /scripts/gen /scripts/continuserv/continuserv /scripts/speculator/speculator diff --git a/README.rst b/README.rst index 6c87201c..c0949d9e 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 2.x virtualenv as follows:: +To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: virtualenv env env/bin/pip install -r scripts/requirements.txt diff --git a/api/check_examples.py b/api/check_examples.py index be0676bb..3ecd5a4c 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in swagger.get('paths', {}).items(): + for path, path_api in list(swagger.get('paths', {}).items()): - for method, request_api in path_api.items(): + for method, request_api in list(path_api.items()): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in responses.items(): + for code, response in list(responses.items()): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 110c4446..6dd9864f 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,16 +94,16 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in api["paths"].items(): + for path, methods in list(api["paths"].items()): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in methods.items(): - if "tags" in spec.keys(): + for method, spec in list(methods.items()): + if "tags" in list(spec.keys()): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec -print "Generating %s" % output_file +print("Generating %s" % output_file) try: os.makedirs(os.path.dirname(output_file)) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2b35f801..0c548611 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) docs_dir = os.path.dirname(script_dir) spec_dir = os.path.join(docs_dir, "specification") tmp_dir = os.path.join(script_dir, "tmp") +changelog_dir = os.path.join(docs_dir, "changelogs") VERBOSE = False @@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs - if isinstance(file_info, basestring): + if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None @@ -194,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section) + outfile.write(section.encode('UTF-8')) """ @@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "r") as f: + with open(path, "rb") as f: lines = f.readlines() replacement = replacement = r'

\n\1' - with open(path, "w") as f: + with open(path, "wb") as f: for line in lines: + line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write(line + "\n") + f.write((line + "\n").encode('UTF-8')) def run_through_template(input_files, set_verbose, substitutions): @@ -297,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in substitutions.items(): + for k, v in list(substitutions.items()): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -357,14 +359,14 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in entry.items() + (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) } return group resolved_files = [] for file_entry in target["files"]: # file_entry is a group id - if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + if isinstance(file_entry, str) and file_entry.startswith("group:"): group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than @@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in file_entry.iteritems(): - if not isinstance(entry, basestring): + for (depth, entry) in list(file_entry.items()): + if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) ) @@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name): return build_target def log(line): - print "gendoc: %s" % line + print("gendoc: %s" % line) def logv(line): if VERBOSE: - print "gendoc:V: %s" % line + print("gendoc:V: %s" % line) def cleanup_env(): @@ -427,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = target_defs["targets"].keys() + targets = list(target_defs["targets"].keys()) log("Building spec [targets=%s]" % targets) @@ -441,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(templated_files.values(), VERBOSE, substitutions) + run_through_template(list(templated_files.values()), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in templated_files.iteritems(): + for target_name, templated_file in list(templated_files.items()): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -479,8 +481,8 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = target_defs["targets"].keys() - print "\n".join(targets) + targets = list(target_defs["targets"].keys()) + print("\n".join(targets)) def extract_major(s): diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d2..156d5351 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in resp.links.values(): + for link in list(resp.links.values()): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/swagger-http-server.py b/scripts/swagger-http-server.py index 5ec00101..06d764aa 100755 --- a/scripts/swagger-http-server.py +++ b/scripts/swagger-http-server.py @@ -19,14 +19,14 @@ import argparse import os -import SimpleHTTPServer -import SocketServer +import http.server +import socketserver # Thanks to http://stackoverflow.com/a/13354482 -class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def end_headers(self): self.send_my_headers() - SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self) + http.server.SimpleHTTPRequestHandler.end_headers(self) def send_my_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -49,7 +49,7 @@ if __name__ == '__main__': os.chdir(args.swagger_dir) - httpd = SocketServer.TCPServer(("localhost", args.port), + httpd = socketserver.TCPServer(("localhost", args.port), MyHTTPRequestHandler) - print "Serving at http://localhost:%i/api-docs.json" % args.port + print("Serving at http://localhost:%i/api-docs.json" % args.port) httpd.serve_forever() diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index da41b31b..f5d83a1e 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sets import Set class AccessKeyStore(object): @@ -22,10 +21,10 @@ class AccessKeyStore(object): if not existing_data: existing_data = {} self.data = existing_data - self.accessed_set = Set() + self.accessed_set = set() def keys(self): - return self.data.keys() + return list(self.data.keys()) def add(self, key, unit_dict): self.data[key] = unit_dict @@ -35,5 +34,5 @@ class AccessKeyStore(object): return self.data[key] def get_unaccessed_set(self): - data_list = Set(self.data.keys()) + data_list = set(self.data.keys()) return data_list - self.accessed_set \ No newline at end of file diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index c541d771..2e5593d6 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -29,7 +29,7 @@ class Sections(object): def log(self, text): if self.debug: - print "batesian:sections: %s" % text + print("batesian:sections: %s" % text) def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -40,7 +40,7 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if isinstance(section, basestring): + if isinstance(section, str): if section_key in section_dict: raise Exception( ("%s : Section %s already exists. It must have been " + @@ -54,8 +54,8 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in section.iteritems(): - if not isinstance(k, basestring) or not isinstance(v, basestring): + for (k, v) in list(section.items()): + if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + "expected the dict elements to be strings but they aren't.") % diff --git a/scripts/templating/batesian/units.py b/scripts/templating/batesian/units.py index 8f748f6d..82cc52f9 100644 --- a/scripts/templating/batesian/units.py +++ b/scripts/templating/batesian/units.py @@ -41,7 +41,7 @@ class Units(object): trace = inspect.stack() if len(trace) > 1 and len(trace[1]) > 2: func_name = trace[1][3] + ":" - print "batesian:units:%s %s" % (func_name, text) + print("batesian:units:%s %s" % (func_name, text)) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -50,7 +50,7 @@ class Units(object): if not func_name.startswith("load_"): continue unit_key = func_name[len("load_"):] - if len(inspect.getargs(func.func_code).args) > 1: + if len(inspect.getargs(func.__code__).args) > 1: unit_dict[unit_key] = func(self.substitutions) else: unit_dict[unit_key] = func() diff --git a/scripts/templating/build.py b/scripts/templating/build.py index d18569b6..de9ead35 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -63,6 +63,7 @@ import sys from textwrap import TextWrapper from matrix_templates.units import TypeTableRow +from functools import reduce def create_from_template(template, sections): @@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} return reduce(max, rowwidths, default if default is not None else default_width) - results = map(colwidth, keys, defaults) + results = list(map(colwidth, keys, defaults)) return results # make Jinja aware of the templates and filters @@ -167,16 +168,16 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: - print "\nValid template variables:" - for key in sections.keys(): + print("\nValid template variables:") + for key in list(sections.keys()): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) sec_info = "%s characters" % len(sections[key]) if sections[key].count("\n") > 0: sec_info += ", %s lines" % sections[key].count("\n") - print " %s" % key - print " %s %s" % (sec_info, sec_text) + print(" %s" % key) + print(" %s %s" % (sec_info, sec_text)) return # check the input files and substitute in sections where required @@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} def process_file(env, sections, filename, output_filename): log("Parsing input template: %s" % filename) - with open(filename, "r") as file_stream: - temp_str = file_stream.read().decode("utf-8") + with open(filename, "rb") as file_stream: + temp_str = file_stream.read().decode('UTF-8') # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. @@ -210,16 +211,16 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): output = output.replace(old, new) - with open(output_filename, "w") as f: - f.write(output.encode("utf-8")) + with open(output_filename, "wb") as f: + f.write(output.encode('UTF-8')) log("Output file for: %s" % output_filename) def log(line): - print "batesian: %s" % line + print("batesian: %s" % line) if __name__ == '__main__': parser = ArgumentParser( diff --git a/scripts/templating/matrix_templates/__init__.py b/scripts/templating/matrix_templates/__init__.py index 6b46192c..b81c5a30 100644 --- a/scripts/templating/matrix_templates/__init__.py +++ b/scripts/templating/matrix_templates/__init__.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sections import MatrixSections -from units import MatrixUnits +from .sections import MatrixSections +from .units import MatrixUnits import os exports = { diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 64e32aa4..71c1acf3 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = self.units.get("swagger_apis").keys() + swagger_groups = list(self.units.get("swagger_apis").keys()) renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in schemas.keys() if k.startswith("m.room.message#") and + k for k in list(schemas.keys()) if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1..40fd96a2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,11 +29,12 @@ import os.path import re import subprocess import sys -import urllib +import urllib.request, urllib.parse, urllib.error import yaml +from functools import reduce matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - range(1, 5), os.path.abspath(__file__)) + list(range(1, 5)), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -124,7 +125,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in schema.items(): + for key, value in list(schema.items()): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -147,7 +148,7 @@ def inherit_parents(obj): # settings defined in the child take priority over the parents, so we # iterate through the parents first, and then overwrite with the settings # from the child. - for p in map(inherit_parents, parents) + [obj]: + for p in list(map(inherit_parents, parents)) + [obj]: # child blats out type, title and description for key in ('type', 'title', 'description'): if p.get(key): @@ -209,7 +210,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in props.keys(): + for key_name in list(props.keys()): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -250,12 +251,12 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables.extend(res["tables"]) logger.debug("Done property %s" % key_name) - except Exception, e: + except Exception as e: e2 = Exception("Error reading property %s.%s: %s" % (obj_title, key_name, str(e))) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows)) @@ -380,7 +381,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in schema['properties'].iteritems(): + for prop_name, prop in list(schema['properties'].items()): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -523,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.quote(example) + "{%s}" % param_name, urllib.parse.quote(example) ) elif param_loc == "query": if type(example) == list: @@ -532,7 +533,7 @@ class MatrixUnits(Units): else: example_query_params.append((param_name, example)) - except Exception, e: + except Exception as e: raise Exception("Error handling parameter %s" % param_name, e) # endfor[param] good_response = None @@ -556,14 +557,14 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in good_response["headers"].iteritems(): + for (header_name, header) in list(good_response["headers"].items()): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.urlencode( + example_query_params) == 0 else "?" + urllib.parse.urlencode( example_query_params) if example_body: endpoint["example"][ @@ -605,17 +606,17 @@ class MatrixUnits(Units): body_tables = req_body_tables[1:] endpoint_data['req_body_tables'].extend(body_tables) - except Exception, e: + except Exception as e: e2 = Exception( "Error decoding body of API endpoint %s %s: %s" % (endpoint_data["method"], endpoint_data["path"], e) ) - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) def load_swagger_apis(self): apis = {} - for path, suffix in HTTP_APIS.items(): + for path, suffix in list(HTTP_APIS.items()): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue @@ -711,12 +712,12 @@ class MatrixUnits(Units): if filename != event_name: examples[event_name] = examples.get(event_name, []) examples[event_name].append(example) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event example "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return examples @@ -730,12 +731,12 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) try: schemata[filename] = self.read_event_schema(filepath) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event schema "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return schemata @@ -871,7 +872,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_branch = "" try: @@ -879,7 +880,7 @@ class MatrixUnits(Units): ['git', 'describe', '--exact-match'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') git_tag = "tag=" + git_tag except subprocess.CalledProcessError: git_tag = "" @@ -888,7 +889,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_commit = "" try: @@ -897,7 +898,7 @@ class MatrixUnits(Units): ['git', 'describe', '--dirty=' + dirty_string, "--all"], stderr=null, cwd=cwd, - ).strip().endswith(dirty_string) + ).strip().decode('UTF-8').endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" except subprocess.CalledProcessError: git_dirty = "" @@ -908,7 +909,7 @@ class MatrixUnits(Units): s for s in (git_branch, git_tag, git_commit, git_dirty,) if s - ).encode("ascii") + ).encode("ascii").decode('ascii') return { "string": git_version, "revision": git_commit From 9277e4c52d70472cd58d7b2bf0ec6329e8228b8a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:32:27 -0600 Subject: [PATCH 02/24] Add initial towncrier support --- .gitignore | 1 + changelogs/client_server/pyproject.toml | 30 ++++++++++++++++++++ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 23 +++++++++++++-- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 changelogs/client_server/pyproject.toml diff --git a/.gitignore b/.gitignore index 58b6099e..a850d2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /templating/out *.pyc *.swp +_rendered.rst diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml new file mode 100644 index 00000000..d5daface --- /dev/null +++ b/changelogs/client_server/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "_rendered.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 37d99dd3..6f7951d6 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,3 +7,4 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 +towncrier == 18.6.0rc1 diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 40fd96a2..0dbf8104 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -832,12 +832,28 @@ class MatrixUnits(Units): path = os.path.join(CHANGELOG_DIR, f) name = f[:-4] + # If there's a directory with the same name, we'll try to generate + # a towncrier changelog and prepend it to the general changelog. + tc_path = os.path.join(CHANGELOG_DIR, name) + tc_lines = [] + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % name) + try: + raw_log = subprocess.check_output( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + stderr=subprocess.PIPE, + cwd=tc_path, + ).strip().decode('UTF-8') + except subprocess.CalledProcessError: + raw_log = "" + tc_lines = raw_log.splitlines() + title_part = None changelog_lines = [] with open(path, "r") as f: lines = f.readlines() prev_line = None - for line in lines: + for line in (tc_lines + lines): if prev_line is None: prev_line = line continue @@ -853,7 +869,10 @@ class MatrixUnits(Units): # then bail out. changelog_lines.pop() break - changelog_lines.append(" " + line) + # Don't generate subheadings (we'll keep the title though) + if re.match("^[-]{3,}$", line.strip()): + continue + changelog_lines.append(" " + line + '\n') changelogs[name] = "".join(changelog_lines) return changelogs From ee9abc5fd76c4e8e62a9a737fbf4ea33afddead5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:41:54 -0600 Subject: [PATCH 03/24] Convert unstable changelog to towncrier --- changelogs/client_server.rst | 70 ------------------- .../client_server/newsfragments/1096.new | 1 + .../newsfragments/1097.clarification | 1 + .../client_server/newsfragments/1104.feature | 1 + .../newsfragments/1106.clarification | 1 + .../newsfragments/1109.clarification | 1 + .../client_server/newsfragments/1110.new | 1 + .../newsfragments/1137.clarification | 1 + .../newsfragments/1139.clarification | 1 + .../client_server/newsfragments/1141.feature | 1 + .../client_server/newsfragments/1142.feature | 1 + .../client_server/newsfragments/1151.feature | 1 + .../newsfragments/1152.clarification | 1 + .../newsfragments/1158.clarification | 3 + .../client_server/newsfragments/1158.feature | 1 + .../client_server/newsfragments/1239.new | 1 + .../newsfragments/1245.clarification | 1 + .../client_server/newsfragments/1263.feature | 1 + .../client_server/newsfragments/1264.feature | 1 + .../client_server/newsfragments/1265.feature | 1 + .../client_server/newsfragments/1274.feature | 1 + .../newsfragments/1329.clarification | 1 + .../client_server/newsfragments/1361.feature | 1 + .../newsfragments/1362.clarification | 1 + .../client_server/newsfragments/1364.feature | 1 + .../newsfragments/1378.clarification | 1 + .../newsfragments/1380.clarification | 1 + .../newsfragments/1381.clarification | 1 + 28 files changed, 29 insertions(+), 70 deletions(-) create mode 100644 changelogs/client_server/newsfragments/1096.new create mode 100644 changelogs/client_server/newsfragments/1097.clarification create mode 100644 changelogs/client_server/newsfragments/1104.feature create mode 100644 changelogs/client_server/newsfragments/1106.clarification create mode 100644 changelogs/client_server/newsfragments/1109.clarification create mode 100644 changelogs/client_server/newsfragments/1110.new create mode 100644 changelogs/client_server/newsfragments/1137.clarification create mode 100644 changelogs/client_server/newsfragments/1139.clarification create mode 100644 changelogs/client_server/newsfragments/1141.feature create mode 100644 changelogs/client_server/newsfragments/1142.feature create mode 100644 changelogs/client_server/newsfragments/1151.feature create mode 100644 changelogs/client_server/newsfragments/1152.clarification create mode 100644 changelogs/client_server/newsfragments/1158.clarification create mode 100644 changelogs/client_server/newsfragments/1158.feature create mode 100644 changelogs/client_server/newsfragments/1239.new create mode 100644 changelogs/client_server/newsfragments/1245.clarification create mode 100644 changelogs/client_server/newsfragments/1263.feature create mode 100644 changelogs/client_server/newsfragments/1264.feature create mode 100644 changelogs/client_server/newsfragments/1265.feature create mode 100644 changelogs/client_server/newsfragments/1274.feature create mode 100644 changelogs/client_server/newsfragments/1329.clarification create mode 100644 changelogs/client_server/newsfragments/1361.feature create mode 100644 changelogs/client_server/newsfragments/1362.clarification create mode 100644 changelogs/client_server/newsfragments/1364.feature create mode 100644 changelogs/client_server/newsfragments/1378.clarification create mode 100644 changelogs/client_server/newsfragments/1380.clarification create mode 100644 changelogs/client_server/newsfragments/1381.clarification diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 32654025..d2ef7f2d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -1,73 +1,3 @@ -Unreleased changes -================== - -- Changes to the API which will be backwards-compatible for clients: - - - New endpoints: - - - ``POST /user_directory/search`` - (`#1096 `_). - - ``GET /rooms/{roomId}/event/{eventId}`` - (`#1110 `_). - - ``POST /delete_devices`` - (`#1239 `_). - - - Sticker messages: - - Add sticker message event definition. - (`#1158 `_). - - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` - (`#1364 `_). - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - - Document the GET version of ``/login`` - (`#1361 `_). - -- Spec clarifications: - - - Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions - to clarify that they relate to intended display size, as opposed to the - intrinsic size of the image file. - (`#1158 `_). - - Mark ``home_server`` return field for ``/login`` and ``/register`` - endpoints as deprecated - (`#1097 `_). - - Fix response format of ``/keys/changes`` endpoint - (`#1106 `_). - - Clarify default values for some fields on the /search API - (`#1109 `_). - - Fix the representation of ``m.presence`` events - (`#1137 `_). - - Clarify that ``m.tag`` ordering is done with numbers, not strings - (`#1139 `_). - - Clarify that ``/account/whoami`` should consider application services - (`#1152 `_). - - Mark ``GET /rooms/{roomId}/members`` as requiring authentication - (`#1245 `_). - - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` - (`#1380 `_). - - Describe ``StateEvent`` for ``/createRoom`` - (`#1329 `_). - - Describe how the ``reason`` is handled for kicks/bans - (`#1362 `_). - - Clarify that clients must leave rooms before forgetting them - (`#1378 `_). - - Clarify the request and result types on ``/search`` - (`#1381 `_). - r0.3.0 ====== diff --git a/changelogs/client_server/newsfragments/1096.new b/changelogs/client_server/newsfragments/1096.new new file mode 100644 index 00000000..50d86879 --- /dev/null +++ b/changelogs/client_server/newsfragments/1096.new @@ -0,0 +1 @@ +``POST /user_directory/search`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1097.clarification b/changelogs/client_server/newsfragments/1097.clarification new file mode 100644 index 00000000..2a7cb93a --- /dev/null +++ b/changelogs/client_server/newsfragments/1097.clarification @@ -0,0 +1 @@ +Mark ``home_server`` return field for ``/login`` and ``/register`` endpoints as deprecated \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1104.feature b/changelogs/client_server/newsfragments/1104.feature new file mode 100644 index 00000000..9b85343f --- /dev/null +++ b/changelogs/client_server/newsfragments/1104.feature @@ -0,0 +1 @@ +Add ``token`` parameter to the ``/keys/query`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1106.clarification b/changelogs/client_server/newsfragments/1106.clarification new file mode 100644 index 00000000..f7a1fe3e --- /dev/null +++ b/changelogs/client_server/newsfragments/1106.clarification @@ -0,0 +1 @@ +Fix response format of ``/keys/changes`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1109.clarification b/changelogs/client_server/newsfragments/1109.clarification new file mode 100644 index 00000000..176d9403 --- /dev/null +++ b/changelogs/client_server/newsfragments/1109.clarification @@ -0,0 +1 @@ +Clarify default values for some fields on the ``/search`` API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1110.new b/changelogs/client_server/newsfragments/1110.new new file mode 100644 index 00000000..e1b80b8b --- /dev/null +++ b/changelogs/client_server/newsfragments/1110.new @@ -0,0 +1 @@ +``GET /rooms/{roomId}/event/{eventId}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1137.clarification b/changelogs/client_server/newsfragments/1137.clarification new file mode 100644 index 00000000..5ad8ec34 --- /dev/null +++ b/changelogs/client_server/newsfragments/1137.clarification @@ -0,0 +1 @@ +Fix the representation of ``m.presence`` events \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1139.clarification b/changelogs/client_server/newsfragments/1139.clarification new file mode 100644 index 00000000..b5193ad3 --- /dev/null +++ b/changelogs/client_server/newsfragments/1139.clarification @@ -0,0 +1 @@ +Clarify that ``m.tag`` ordering is done with numbers, not strings \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1141.feature b/changelogs/client_server/newsfragments/1141.feature new file mode 100644 index 00000000..da041819 --- /dev/null +++ b/changelogs/client_server/newsfragments/1141.feature @@ -0,0 +1 @@ +Add the room visibility options for the room directory \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1142.feature b/changelogs/client_server/newsfragments/1142.feature new file mode 100644 index 00000000..0a0842c4 --- /dev/null +++ b/changelogs/client_server/newsfragments/1142.feature @@ -0,0 +1 @@ +Add spec for ignoring users \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1151.feature b/changelogs/client_server/newsfragments/1151.feature new file mode 100644 index 00000000..8875812b --- /dev/null +++ b/changelogs/client_server/newsfragments/1151.feature @@ -0,0 +1 @@ +Add the ``/register/available`` endpoint for username availability \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1152.clarification b/changelogs/client_server/newsfragments/1152.clarification new file mode 100644 index 00000000..bbecc9b2 --- /dev/null +++ b/changelogs/client_server/newsfragments/1152.clarification @@ -0,0 +1 @@ +Clarify that ``/account/whoami`` should consider application services \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.clarification b/changelogs/client_server/newsfragments/1158.clarification new file mode 100644 index 00000000..dc1f6d14 --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.clarification @@ -0,0 +1,3 @@ +Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions +to clarify that they relate to intended display size, as opposed to the +intrinsic size of the image file. \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.feature b/changelogs/client_server/newsfragments/1158.feature new file mode 100644 index 00000000..a55df4fb --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.feature @@ -0,0 +1 @@ +Add sticker messages \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1239.new b/changelogs/client_server/newsfragments/1239.new new file mode 100644 index 00000000..9bcf357d --- /dev/null +++ b/changelogs/client_server/newsfragments/1239.new @@ -0,0 +1 @@ +``POST /delete_devices`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1245.clarification b/changelogs/client_server/newsfragments/1245.clarification new file mode 100644 index 00000000..e0a63834 --- /dev/null +++ b/changelogs/client_server/newsfragments/1245.clarification @@ -0,0 +1 @@ +Mark ``GET /rooms/{roomId}/members`` as requiring authentication \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1263.feature b/changelogs/client_server/newsfragments/1263.feature new file mode 100644 index 00000000..04964a7d --- /dev/null +++ b/changelogs/client_server/newsfragments/1263.feature @@ -0,0 +1 @@ +Document ``/logout/all`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1264.feature b/changelogs/client_server/newsfragments/1264.feature new file mode 100644 index 00000000..9cb06f71 --- /dev/null +++ b/changelogs/client_server/newsfragments/1264.feature @@ -0,0 +1 @@ +Add report content API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1265.feature b/changelogs/client_server/newsfragments/1265.feature new file mode 100644 index 00000000..1e270fa8 --- /dev/null +++ b/changelogs/client_server/newsfragments/1265.feature @@ -0,0 +1 @@ +Add ``allow_remote`` to the content repo to avoid routing loops \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1274.feature b/changelogs/client_server/newsfragments/1274.feature new file mode 100644 index 00000000..d4958131 --- /dev/null +++ b/changelogs/client_server/newsfragments/1274.feature @@ -0,0 +1 @@ +Document `highlights` field in /search response \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1329.clarification b/changelogs/client_server/newsfragments/1329.clarification new file mode 100644 index 00000000..970d3d94 --- /dev/null +++ b/changelogs/client_server/newsfragments/1329.clarification @@ -0,0 +1 @@ +Describe ``StateEvent`` for ``/createRoom`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1361.feature b/changelogs/client_server/newsfragments/1361.feature new file mode 100644 index 00000000..b1d4c2f1 --- /dev/null +++ b/changelogs/client_server/newsfragments/1361.feature @@ -0,0 +1 @@ +Document the GET version of ``/login`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1362.clarification b/changelogs/client_server/newsfragments/1362.clarification new file mode 100644 index 00000000..1deb4500 --- /dev/null +++ b/changelogs/client_server/newsfragments/1362.clarification @@ -0,0 +1 @@ +Describe how the ``reason`` is handled for kicks/bans \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1364.feature b/changelogs/client_server/newsfragments/1364.feature new file mode 100644 index 00000000..733d6a1f --- /dev/null +++ b/changelogs/client_server/newsfragments/1364.feature @@ -0,0 +1 @@ +Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1378.clarification b/changelogs/client_server/newsfragments/1378.clarification new file mode 100644 index 00000000..f952428b --- /dev/null +++ b/changelogs/client_server/newsfragments/1378.clarification @@ -0,0 +1 @@ +Clarify that clients must leave rooms before forgetting them \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1380.clarification b/changelogs/client_server/newsfragments/1380.clarification new file mode 100644 index 00000000..490a9a43 --- /dev/null +++ b/changelogs/client_server/newsfragments/1380.clarification @@ -0,0 +1 @@ +Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1381.clarification b/changelogs/client_server/newsfragments/1381.clarification new file mode 100644 index 00000000..e5e599ac --- /dev/null +++ b/changelogs/client_server/newsfragments/1381.clarification @@ -0,0 +1 @@ +Clarify the request and result types on ``/search`` \ No newline at end of file From c2f85788ded8970cad38bc261314ae148c492e20 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:54:14 -0600 Subject: [PATCH 04/24] Add documentation about how to add to the changelog --- CONTRIBUTING.rst | 35 +++++++++++++++++++++++++++++++++++ scripts/gendoc.py | 3 ++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a7416b1..c592cf02 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,6 +69,41 @@ For such changes, please do just open a `pull request`_. .. _pull request: https://help.github.com/articles/about-pull-requests + +Adding to the changelog +~~~~~~~~~~~~~~~~~~~~~~~ + +Currently only changes to the client-server API need to end up in a changelog. The +other APIs are not yet stable and therefore do not have a changelog. Adding to the +changelog can only be done after you've opened your pull request, so be sure to do +that first. + +The changelog is managed by Towncrier (https://github.com/hawkowl/towncrier) in the +form of "news fragments". The news fragments for the client-server API are stored +under ``changelogs/client_server/newsfragments``. + +To create a changelog entry, create a file named in the format ``prNumber.type`` in +the ``newsfragments`` directory. The ``type`` can be one of the following: + +* ``new`` - Used when adding new endpoints. Please have the file contents be the + method and route being added, surrounded in RST code tags. For example: ``POST + /accounts/whoami`` + +* ``feature`` - Used when adding backwards-compatible changes to the API. + +* ``clarification`` - Used when an area of the spec is being improved upon and does + not change or introduce any functionality. + +* ``breaking`` - Used when the change is not backwards compatible. + +* ``deprecation`` - Used when deprecating something + +All news fragments must have a brief summary explaining the change in the contents +of the file. + +Changes that do not change the spec, such as changes to the build script, formatting, +CSS, etc should not get a news fragment. + Sign off -------- diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 0c548611..31ee9e69 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,7 +405,8 @@ def logv(line): def cleanup_env(): - shutil.rmtree(tmp_dir) + #shutil.rmtree(tmp_dir) + pass def mkdirp(d) : From 34089e302c8415a34f69a200f4aee689fd97a0c8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:06:08 -0600 Subject: [PATCH 05/24] Add general documentation for maintainers --- changelogs/README.md | 55 +++++++++++++++++++++++++ changelogs/client_server/pyproject.toml | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelogs/README.md diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 00000000..10d69cb6 --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,55 @@ + + +# Changelogs + +[Towncrier](https://github.com/hawkowl/towncrier) is used to manage the changelog and +keep it up to date. Because of this, updating a changelog is really easy. + +## How to update a changelog when releasing an API + +1. Ensure you're in your Python 3 virtual environment +2. `cd` your way to the API you're releasing (eg: `cd changelogs/client_server`) +3. Run `towncrier --version "r0.4.0" --name "client-server" --yes` substituting the + variables as approprite. Note that `--name` is required although the value is ignored. +4. Commit the changes and finish the release process. + +## How to prepare a changelog for a new API + +For this example, we're going to pretend that the `server_server` API doesn't exist. + +1. Create the file `changelogs/server_server.rst` +2. Create the folder `changelogs/server_server` +3. In the new folder, create a `pyproject.toml` file with these contents: + ```toml + [tool.towncrier] + filename = "../server_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true + ``` +4. Use the changelog in whatever RST documents you need diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml index d5daface..8fa3f6b5 100644 --- a/changelogs/client_server/pyproject.toml +++ b/changelogs/client_server/pyproject.toml @@ -1,5 +1,5 @@ [tool.towncrier] - filename = "_rendered.rst" + filename = "../client_server.rst" directory = "newsfragments" issue_format = "`#{issue} `_" title_format = "{version}" From 63b926a4aaae18293426cc10efa658a8644b7e42 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:19:04 -0600 Subject: [PATCH 06/24] Make Travis CI use python 3 --- .travis.yml | 4 ++++ scripts/test-and-build.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d84f237..98dd14b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ sudo: false git: depth: 1 +before_install: +- sudo apt-get update +- sudo apt-get install python3 + # test-and-build does the installation, so tell travis to skip the # installation step install: true diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 7794f826..e459a370 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv env +virtualenv -p python3 env . env/bin/activate pip install -r scripts/requirements.txt From f68c82bd6c3d8959004ee564ae9dce6769aa99a8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:32:57 -0600 Subject: [PATCH 07/24] Don't let the newsfragments wink out of existence --- changelogs/README.md | 2 +- changelogs/client_server/newsfragments/.gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelogs/client_server/newsfragments/.gitignore diff --git a/changelogs/README.md b/changelogs/README.md index 10d69cb6..a5fb1fb7 100644 --- a/changelogs/README.md +++ b/changelogs/README.md @@ -52,4 +52,4 @@ For this example, we're going to pretend that the `server_server` API doesn't ex name = "Spec Clarifications" showcontent = true ``` -4. Use the changelog in whatever RST documents you need +4. Create a `.gitignore` in `changelogs/server_server/newsfragments` with the contents `!.gitignore` diff --git a/changelogs/client_server/newsfragments/.gitignore b/changelogs/client_server/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/client_server/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file From a4f8e0807acee23f58b2a9e4ae2b0b0ba9f21cfa Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:41:20 -0600 Subject: [PATCH 08/24] Print the python version being used in the CI build. --- scripts/test-and-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index e459a370..710b03dd 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -6,6 +6,11 @@ cd `dirname $0`/.. virtualenv -p python3 env . env/bin/activate + +# Print out the python versions for debugging purposes +python --version +pip --version + pip install -r scripts/requirements.txt # do sanity checks on the examples and swagger From 08f69128259bc3113614018e04b9342850388d2c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:50:26 -0600 Subject: [PATCH 09/24] Ask Travis to install 3.5 explicitly --- .travis.yml | 2 +- scripts/test-and-build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98dd14b8..013efdd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ git: before_install: - sudo apt-get update -- sudo apt-get install python3 +- sudo apt-get install python3.5 # test-and-build does the installation, so tell travis to skip the # installation step diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 710b03dd..397b620e 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3 env +virtualenv -p python3.5 env . env/bin/activate # Print out the python versions for debugging purposes From c65c6e2e70795dabd7717e195a72a39704e8a014 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:25:34 -0600 Subject: [PATCH 10/24] Add changelog entries removed in the merge from master --- changelogs/client_server/newsfragments/1371.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1371.clarification diff --git a/changelogs/client_server/newsfragments/1371.clarification b/changelogs/client_server/newsfragments/1371.clarification new file mode 100644 index 00000000..88552fcd --- /dev/null +++ b/changelogs/client_server/newsfragments/1371.clarification @@ -0,0 +1 @@ +Mark ``GET /presence/{userId}/status`` as requiring authentication \ No newline at end of file From 7ada96020677b9df88422ac38b2b82754d974321 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:28:27 -0600 Subject: [PATCH 11/24] Fix the Travis CI build to work on 3.5 --- .travis.yml | 9 +++------ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 7 ++++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 013efdd9..27e5cf20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,10 @@ sudo: false git: depth: 1 -before_install: +install: - sudo apt-get update -- sudo apt-get install python3.5 - -# test-and-build does the installation, so tell travis to skip the -# installation step -install: true +- sudo apt-get install python3.5 python3.5-dev +- sudo pip install --upgrade virtualenv script: - ./scripts/test-and-build.sh diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 6f7951d6..1f92c3dc 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -8,3 +8,4 @@ jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 towncrier == 18.6.0rc1 +six >= 1.11.0 \ No newline at end of file diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 0dbf8104..acd85d66 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,9 +29,10 @@ import os.path import re import subprocess import sys -import urllib.request, urllib.parse, urllib.error import yaml +import requests.utils from functools import reduce +from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), list(range(1, 5)), os.path.abspath(__file__)) @@ -524,7 +525,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.parse.quote(example) + "{%s}" % param_name, requests.utils.requote_uri(example) ) elif param_loc == "query": if type(example) == list: @@ -564,7 +565,7 @@ class MatrixUnits(Units): ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.parse.urlencode( + example_query_params) == 0 else "?" + urlencode( example_query_params) if example_body: endpoint["example"][ From 550f95570b327e8f57e832866427049603f48b06 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 15:17:28 -0600 Subject: [PATCH 12/24] Fix regex for some versions of python --- scripts/gendoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 31ee9e69..25fd4577 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -283,12 +283,12 @@ def addAnchors(path): with open(path, "rb") as f: lines = f.readlines() - replacement = replacement = r'

\n\1' + replacement = r'

\n\1' with open(path, "wb") as f: for line in lines: line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) - line = re.sub(r'(
)', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) f.write((line + "\n").encode('UTF-8')) From e9f7d2096e100c27c530ec4660670bdb9f8605a4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 08:39:34 -0600 Subject: [PATCH 13/24] Add changelog entry for CORS --- changelogs/client_server/newsfragments/1365.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1365.feature diff --git a/changelogs/client_server/newsfragments/1365.feature b/changelogs/client_server/newsfragments/1365.feature new file mode 100644 index 00000000..d2864e96 --- /dev/null +++ b/changelogs/client_server/newsfragments/1365.feature @@ -0,0 +1 @@ +Document the CORS/preflight headers \ No newline at end of file From 9e0fafbcd4436d3d245a942117eccb1d2396261d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:13:22 -0600 Subject: [PATCH 14/24] Remove debugging line --- scripts/gendoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 25fd4577..af613b68 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,8 +405,7 @@ def logv(line): def cleanup_env(): - #shutil.rmtree(tmp_dir) - pass + shutil.rmtree(tmp_dir) def mkdirp(d) : From ebc7db12fbb9e5ff0f6193605da8182710d0937c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:52:17 -0600 Subject: [PATCH 15/24] Remove extraneous list casting --- api/check_examples.py | 6 +++--- scripts/dump-swagger.py | 6 +++--- scripts/gendoc.py | 16 ++++++++-------- scripts/proposals.py | 2 +- scripts/templating/batesian/__init__.py | 2 +- scripts/templating/batesian/sections.py | 2 +- scripts/templating/build.py | 4 ++-- scripts/templating/matrix_templates/sections.py | 4 ++-- scripts/templating/matrix_templates/units.py | 12 ++++++------ 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index 3ecd5a4c..be0676bb 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in list(swagger.get('paths', {}).items()): + for path, path_api in swagger.get('paths', {}).items(): - for method, request_api in list(path_api.items()): + for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in list(responses.items()): + for code, response in responses.items(): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 6dd9864f..7994324f 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,11 +94,11 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in list(api["paths"].items()): + for path, methods in api["paths"].items(): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in list(methods.items()): - if "tags" in list(spec.keys()): + for method, spec in methods.items(): + if "tags" in spec.keys(): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec diff --git a/scripts/gendoc.py b/scripts/gendoc.py index af613b68..16c40af5 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -299,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in list(substitutions.items()): + for k, v in substitutions.items(): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -359,7 +359,7 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) + (rel_depth + depth): v for (rel_depth, v) in entry.items() } return group @@ -378,7 +378,7 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in list(file_entry.items()): + for (depth, entry) in file_entry.items(): if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) @@ -429,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() log("Building spec [targets=%s]" % targets) @@ -443,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(list(templated_files.values()), VERBOSE, substitutions) + run_through_template(templated_files.values(), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in list(templated_files.items()): + for target_name, templated_file in templated_files.items(): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -481,7 +481,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() print("\n".join(targets)) diff --git a/scripts/proposals.py b/scripts/proposals.py index 156d5351..b489f0d2 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in list(resp.links.values()): + for link in resp.links.values(): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index f5d83a1e..e901590f 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -24,7 +24,7 @@ class AccessKeyStore(object): self.accessed_set = set() def keys(self): - return list(self.data.keys()) + return self.data.keys() def add(self, key, unit_dict): self.data[key] = unit_dict diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index 2e5593d6..18a622f6 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -54,7 +54,7 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in list(section.items()): + for (k, v) in section.items(): if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + diff --git a/scripts/templating/build.py b/scripts/templating/build.py index de9ead35..fae4db56 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -169,7 +169,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: print("\nValid template variables:") - for key in list(sections.keys()): + for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) @@ -211,7 +211,7 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): output = output.replace(old, new) with open(output_filename, "wb") as f: diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 71c1acf3..64e32aa4 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = list(self.units.get("swagger_apis").keys()) + swagger_groups = self.units.get("swagger_apis").keys() renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in list(schemas.keys()) if k.startswith("m.room.message#") and + k for k in schemas.keys() if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index acd85d66..63a4e5a8 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -35,7 +35,7 @@ from functools import reduce from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - list(range(1, 5)), os.path.abspath(__file__)) + range(1, 5), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -126,7 +126,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in list(schema.items()): + for key, value in schema.items(): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -211,7 +211,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in list(props.keys()): + for key_name in props.keys(): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -382,7 +382,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in list(schema['properties'].items()): + for prop_name, prop in schema['properties'].items(): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -558,7 +558,7 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in list(good_response["headers"].items()): + for (header_name, header) in good_response["headers"].items(): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), @@ -617,7 +617,7 @@ class MatrixUnits(Units): def load_swagger_apis(self): apis = {} - for path, suffix in list(HTTP_APIS.items()): + for path, suffix in HTTP_APIS.items(): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue From 905ef6daddbaeff457c9f265d0d2bf64332c558b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:57:21 -0600 Subject: [PATCH 16/24] Have the towncrier generator complain if the output looks wrong --- scripts/templating/matrix_templates/units.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 63a4e5a8..9b3d621d 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -845,7 +845,14 @@ class MatrixUnits(Units): stderr=subprocess.PIPE, cwd=tc_path, ).strip().decode('UTF-8') - except subprocess.CalledProcessError: + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("Unreleased Changes"): + logger.error(raw_log) + raw_log = "" + except subprocess.CalledProcessError as e: + logger.error(e) raw_log = "" tc_lines = raw_log.splitlines() From a8461e647f2c5c6bf8cdb4f5157736ed2589948a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 08:35:55 -0600 Subject: [PATCH 17/24] Improve the error handling for towncrier The changelog shows up via stdout, everything else via stderr. We dump as much information as we can into the changelog to make debugging errors easier. --- scripts/templating/matrix_templates/units.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 9b3d621d..760aeb6b 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -839,21 +839,28 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - try: - raw_log = subprocess.check_output( - ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], - stderr=subprocess.PIPE, - cwd=tc_path, - ).strip().decode('UTF-8') + p = subprocess.run( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + cwd=tc_path, + check=False, # We'll manually check the exit code + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(p.stdout.decode('UTF-8')) + logger.error(p.stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = p.stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke if not raw_log.startswith("Unreleased Changes"): + logger.error("Towncrier appears to have failed to generate a changelog") logger.error(raw_log) raw_log = "" - except subprocess.CalledProcessError as e: - logger.error(e) - raw_log = "" tc_lines = raw_log.splitlines() title_part = None From af7460088fa756eb2b3afff2949f5ee281c2e2e0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:07:09 -0600 Subject: [PATCH 18/24] Use six for url quoting --- scripts/templating/matrix_templates/units.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 760aeb6b..12db409f 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -30,9 +30,8 @@ import re import subprocess import sys import yaml -import requests.utils from functools import reduce -from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode, quote matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), range(1, 5), os.path.abspath(__file__)) @@ -525,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, requests.utils.requote_uri(example) + "{%s}" % param_name, quote(example) ) elif param_loc == "query": if type(example) == list: From 8a4ba8c5cac07011159855735e96948f33787e08 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:16:32 -0600 Subject: [PATCH 19/24] Use Popen rather than run() --- scripts/templating/matrix_templates/units.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 12db409f..cd563346 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -838,21 +838,21 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - p = subprocess.run( + p = subprocess.Popen( ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], cwd=tc_path, - check=False, # We'll manually check the exit code stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) + stdout, stderr = p.communicate() if p.returncode != 0: # Something broke - dump as much information as we can logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(p.stdout.decode('UTF-8')) - logger.error(p.stderr.decode('UTF-8')) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) raw_log = "" else: - raw_log = p.stdout.decode('UTF-8') + raw_log = stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke From 527a5a2fdccd72d38a7b7d3e621b31b9128a23c3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:18:46 -0600 Subject: [PATCH 20/24] Require sudo see also: https://github.com/travis-ci/travis-ci/issues/9875 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27e5cf20..68fc7743 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.8 -sudo: false +sudo: required # we only need a single git commit git: From 1e500fd00563d52006807886cb223be107482d27 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 13:18:27 -0600 Subject: [PATCH 21/24] Add newsfragment for third party network endpoints Specifying multiple entries for the same PR would be covered under: https://github.com/hawkowl/towncrier/pull/119 --- changelogs/client_server/newsfragments/1353.new | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1353.new diff --git a/changelogs/client_server/newsfragments/1353.new b/changelogs/client_server/newsfragments/1353.new new file mode 100644 index 00000000..0af0c520 --- /dev/null +++ b/changelogs/client_server/newsfragments/1353.new @@ -0,0 +1 @@ +``GET /thirdparty/*`` Endpoints From baf19cc741071bf6e5b5c3235f56a9cd99488ae8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 10:05:14 -0600 Subject: [PATCH 22/24] Add newsfragment for 1379 --- changelogs/client_server/newsfragments/1379.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1379.clarification diff --git a/changelogs/client_server/newsfragments/1379.clarification b/changelogs/client_server/newsfragments/1379.clarification new file mode 100644 index 00000000..122b3900 --- /dev/null +++ b/changelogs/client_server/newsfragments/1379.clarification @@ -0,0 +1 @@ +Document guest access in ``/createRoom`` presets From cfac4a1ab38ed05fc677c999b8ab379e1ff97eca Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 08:48:31 -0600 Subject: [PATCH 23/24] Use towncrier 18.6.0 final --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1f92c3dc..2a7d7ff8 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,5 +7,5 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 -towncrier == 18.6.0rc1 +towncrier == 18.6.0 six >= 1.11.0 \ No newline at end of file From 98c607bff5e59d9d1db4a5af34fb197ac996d428 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 10:27:01 -0600 Subject: [PATCH 24/24] Advertise python 3.4 support and use that in Travis CI We also don't need to update virtualenv, or use python3.5 specific packages. --- .travis.yml | 5 ++--- README.rst | 4 ++-- scripts/test-and-build.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68fc7743..9e9363af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,8 @@ git: depth: 1 install: -- sudo apt-get update -- sudo apt-get install python3.5 python3.5-dev -- sudo pip install --upgrade virtualenv + - sudo apt-get update + - sudo apt-get install python3 python3-dev script: - ./scripts/test-and-build.sh diff --git a/README.rst b/README.rst index c0949d9e..b8847bfb 100644 --- a/README.rst +++ b/README.rst @@ -41,9 +41,9 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: +To use the scripts, it is best to create a Python 3.4+ virtualenv as follows:: - virtualenv env + virtualenv -p python3 env env/bin/pip install -r scripts/requirements.txt (Benjamin Synders has contributed a script for `Nix`_ users, which can be diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 397b620e..710b03dd 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3.5 env +virtualenv -p python3 env . env/bin/activate # Print out the python versions for debugging purposes