wholetrans/docs-matrix-spec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'matrix-org/master' into travis/s2s/transactions-swagger

Browse source
This commit is contained in:
Travis Ralston 2018-07-18 10:59:39 -06:00
commit e226b60c7f
93 changed files with 1256 additions and 263 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/dump-swagger.py
View file

@ -103,7 +103,7 @@ for filename in os.listdir(cs_api_dir):
output["paths"][path] = {}
output["paths"][path][method] = spec
print "Generating %s" % output_file
print("Generating %s" % output_file)
try:
os.makedirs(os.path.dirname(output_file))

30
scripts/gendoc.py
View file

@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__))
docs_dir = os.path.dirname(script_dir)
spec_dir = os.path.join(docs_dir, "specification")
tmp_dir = os.path.join(script_dir, "tmp")
changelog_dir = os.path.join(docs_dir, "changelogs")
VERBOSE = False
@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles):
def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
# string are file paths to RST blobs
if isinstance(file_info, basestring):
if isinstance(file_info, str):
log("%s %s" % (">" * (1 + title_level), file_info))
with open(os.path.join(spec_dir, file_info), "r") as f:
rst = None
@ -194,7 +195,7 @@ def build_spec(target, out_filename):
spec_dir=spec_dir,
adjust_titles=True
)
outfile.write(section)
outfile.write(section.encode('UTF-8'))
"""
@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets):
def addAnchors(path):
log("add anchors %s" % path)
with open(path, "r") as f:
with open(path, "rb") as f:
lines = f.readlines()
replacement = replacement = r'<p><a class="anchor" id="\3"></a></p>\n\1'
with open(path, "w") as f:
replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1'
with open(path, "wb") as f:
for line in lines:
line = line.decode("UTF-8")
line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
line = re.sub(r'(<div class="section" (id)="(.*?)">)', replacement, line.rstrip())
f.write(line + "\n")
line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip())
f.write((line + "\n").encode('UTF-8'))
def run_through_template(input_files, set_verbose, substitutions):
@ -364,7 +366,7 @@ def get_build_target(all_targets, target_name):
resolved_files = []
for file_entry in target["files"]:
# file_entry is a group id
if isinstance(file_entry, basestring) and file_entry.startswith("group:"):
if isinstance(file_entry, str) and file_entry.startswith("group:"):
group = get_group(file_entry, 0)
# The group may be resolved to a list of file entries, in which case
# we want to extend the array to insert each of them rather than
@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name):
# file_entry is a dict which has more file entries as values
elif isinstance(file_entry, dict):
resolved_entry = {}
for (depth, entry) in file_entry.iteritems():
if not isinstance(entry, basestring):
for (depth, entry) in file_entry.items():
if not isinstance(entry, str):
raise Exception(
"Double-nested depths are not supported. Entry: %s" % (file_entry,)
)
@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name):
return build_target
def log(line):
print "gendoc: %s" % line
print("gendoc: %s" % line)
def logv(line):
if VERBOSE:
print "gendoc:V: %s" % line
print("gendoc:V: %s" % line)
def cleanup_env():
@ -445,7 +447,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions):
stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css"))
for target_name, templated_file in templated_files.iteritems():
for target_name, templated_file in templated_files.items():
target = target_defs["targets"].get(target_name)
version_label = None
if target:
@ -480,7 +482,7 @@ def list_targets():
with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file:
target_defs = yaml.load(targ_file.read())
targets = target_defs["targets"].keys()
print "\n".join(targets)
print("\n".join(targets))
def extract_major(s):

11
scripts/proposals.py
View file

@ -12,13 +12,18 @@ authors = set()
prs = set()
def getpage(url, page):
resp = requests.get(url + str(page))
url = url + str(page)
resp = requests.get(url)
for link in resp.links.values():
if link['rel'] == 'last':
pagecount = re.search('page=(.+?)', link['url']).group(1)
return resp.json()
val = resp.json()
if not isinstance(val, list):
print(val) # Just dump the raw (likely error) response to the log
raise Exception("Error calling %s" % url)
return val
def getbylabel(label):
pagecount = 1
@ -27,7 +32,7 @@ def getbylabel(label):
print(urlbase)
json.extend(getpage(urlbase, 1))
for page in range(2, int(pagecount) + 1):
getpage(urlbase, page)
json.extend(getpage(urlbase, page))
return json

2
scripts/requirements.txt
View file

@ -7,3 +7,5 @@ Jinja2 >= 2.9.6
jsonschema >= 2.6.0
PyYAML >= 3.12
requests >= 2.18.4
towncrier == 18.6.0
six >= 1.11.0

12
scripts/swagger-http-server.py
View file

@ -19,14 +19,14 @@
import argparse
import os
import SimpleHTTPServer
import SocketServer
import http.server
import socketserver
# Thanks to http://stackoverflow.com/a/13354482
class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def end_headers(self):
self.send_my_headers()
SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self)
http.server.SimpleHTTPRequestHandler.end_headers(self)
def send_my_headers(self):
self.send_header("Access-Control-Allow-Origin", "*")
@ -49,7 +49,7 @@ if __name__ == '__main__':
os.chdir(args.swagger_dir)
httpd = SocketServer.TCPServer(("localhost", args.port),
httpd = socketserver.TCPServer(("localhost", args.port),
MyHTTPRequestHandler)
print "Serving at http://localhost:%i/api-docs.json" % args.port
print("Serving at http://localhost:%i/api-docs.json" % args.port)
httpd.serve_forever()

5
scripts/templating/batesian/__init__.py
View file

@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sets import Set
class AccessKeyStore(object):
@ -22,7 +21,7 @@ class AccessKeyStore(object):
if not existing_data:
existing_data = {}
self.data = existing_data
self.accessed_set = Set()
self.accessed_set = set()
def keys(self):
return self.data.keys()
@ -35,5 +34,5 @@ class AccessKeyStore(object):
return self.data[key]
def get_unaccessed_set(self):
data_list = Set(self.data.keys())
data_list = set(self.data.keys())
return data_list - self.accessed_set

8
scripts/templating/batesian/sections.py
View file

@ -29,7 +29,7 @@ class Sections(object):
def log(self, text):
if self.debug:
print "batesian:sections: %s" % text
print("batesian:sections: %s" % text)
def get_sections(self):
render_list = inspect.getmembers(self, predicate=inspect.ismethod)
@ -40,7 +40,7 @@ class Sections(object):
section_key = func_name[len("render_"):]
self.log("Generating section '%s'" % section_key)
section = func()
if isinstance(section, basestring):
if isinstance(section, str):
if section_key in section_dict:
raise Exception(
("%s : Section %s already exists. It must have been " +
@ -54,8 +54,8 @@ class Sections(object):
)
elif isinstance(section, dict):
self.log(" Generated multiple sections:")
for (k, v) in section.iteritems():
if not isinstance(k, basestring) or not isinstance(v, basestring):
for (k, v) in section.items():
if not isinstance(k, str) or not isinstance(v, str):
raise Exception(
("Method %s returned multiple sections as a dict but " +
"expected the dict elements to be strings but they aren't.") %

4
scripts/templating/batesian/units.py
View file

@ -41,7 +41,7 @@ class Units(object):
trace = inspect.stack()
if len(trace) > 1 and len(trace[1]) > 2:
func_name = trace[1][3] + ":"
print "batesian:units:%s %s" % (func_name, text)
print("batesian:units:%s %s" % (func_name, text))
def get_units(self, debug=False):
unit_list = inspect.getmembers(self, predicate=inspect.ismethod)
@ -50,7 +50,7 @@ class Units(object):
if not func_name.startswith("load_"):
continue
unit_key = func_name[len("load_"):]
if len(inspect.getargs(func.func_code).args) > 1:
if len(inspect.getargs(func.__code__).args) > 1:
unit_dict[unit_key] = func(self.substitutions)
else:
unit_dict[unit_key] = func()

19
scripts/templating/build.py
View file

@ -63,6 +63,7 @@ import sys
from textwrap import TextWrapper
from matrix_templates.units import TypeTableRow
from functools import reduce
def create_from_template(template, sections):
@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
return reduce(max, rowwidths,
default if default is not None else default_width)
results = map(colwidth, keys, defaults)
results = list(map(colwidth, keys, defaults))
return results
# make Jinja aware of the templates and filters
@ -167,7 +168,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
# print out valid section keys if no file supplied
if not files:
print "\nValid template variables:"
print("\nValid template variables:")
for key in sections.keys():
sec_text = "" if (len(sections[key]) > 75) else (
"(Value: '%s')" % sections[key]
@ -175,8 +176,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
sec_info = "%s characters" % len(sections[key])
if sections[key].count("\n") > 0:
sec_info += ", %s lines" % sections[key].count("\n")
print " %s" % key
print " %s %s" % (sec_info, sec_text)
print(" %s" % key)
print(" %s %s" % (sec_info, sec_text))
return
# check the input files and substitute in sections where required
@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}
def process_file(env, sections, filename, output_filename):
log("Parsing input template: %s" % filename)
with open(filename, "r") as file_stream:
temp_str = file_stream.read().decode("utf-8")
with open(filename, "rb") as file_stream:
temp_str = file_stream.read().decode('UTF-8')
# do sanity checking on the template to make sure they aren't reffing things
# which will never be replaced with a section.
@ -213,13 +214,13 @@ def process_file(env, sections, filename, output_filename):
for old, new in substitutions.items():
output = output.replace(old, new)
with open(output_filename, "w") as f:
f.write(output.encode("utf-8"))
with open(output_filename, "wb") as f:
f.write(output.encode('UTF-8'))
log("Output file for: %s" % output_filename)
def log(line):
print "batesian: %s" % line
print("batesian: %s" % line)
if __name__ == '__main__':
parser = ArgumentParser(

4
scripts/templating/matrix_templates/__init__.py
View file

@ -11,8 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sections import MatrixSections
from units import MatrixUnits
from .sections import MatrixSections
from .units import MatrixUnits
import os
exports = {

78
scripts/templating/matrix_templates/units.py
View file

@ -29,8 +29,9 @@ import os.path
import re
import subprocess
import sys
import urllib
import yaml
from functools import reduce
from six.moves.urllib.parse import urlencode, quote
matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc),
range(1, 5), os.path.abspath(__file__))
@ -151,7 +152,7 @@ def inherit_parents(obj):
# settings defined in the child take priority over the parents, so we
# iterate through the parents first, and then overwrite with the settings
# from the child.
for p in map(inherit_parents, parents) + [obj]:
for p in list(map(inherit_parents, parents)) + [obj]:
# child blats out type, title and description
for key in ('type', 'title', 'description'):
if p.get(key):
@ -254,12 +255,12 @@ def get_json_schema_object_fields(obj, enforce_title=False):
tables.extend(res["tables"])
logger.debug("Done property %s" % key_name)
except Exception, e:
except Exception as e:
e2 = Exception("Error reading property %s.%s: %s" %
(obj_title, key_name, str(e)))
# throw the new exception with the old stack trace, so that
# we don't lose information about where the error occurred.
raise e2, None, sys.exc_info()[2]
raise e2.with_traceback(sys.exc_info()[2])
tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows))
@ -397,7 +398,7 @@ def get_example_for_schema(schema):
if 'properties' not in schema:
raise Exception('"object" property has neither properties nor example')
res = OrderedDict()
for prop_name, prop in schema['properties'].iteritems():
for prop_name, prop in schema['properties'].items():
logger.debug("Parsing property %r" % prop_name)
prop_example = get_example_for_schema(prop)
res[prop_name] = prop_example
@ -549,7 +550,7 @@ class MatrixUnits(Units):
if param_loc == "path":
path_template = path_template.replace(
"{%s}" % param_name, urllib.quote(example)
"{%s}" % param_name, quote(example)
)
elif param_loc == "query":
if type(example) == list:
@ -558,7 +559,7 @@ class MatrixUnits(Units):
else:
example_query_params.append((param_name, example))
except Exception, e:
except Exception as e:
raise Exception("Error handling parameter %s" % param_name, e)
# endfor[param]
good_response = None
@ -582,14 +583,14 @@ class MatrixUnits(Units):
)
if "headers" in good_response:
headers = TypeTable()
for (header_name, header) in good_response["headers"].iteritems():
for (header_name, header) in good_response["headers"].items():
headers.add_row(
TypeTableRow(key=header_name, title=header["type"],
desc=header["description"]),
)
endpoint["res_headers"] = headers
query_string = "" if len(
example_query_params) == 0 else "?" + urllib.urlencode(
example_query_params) == 0 else "?" + urlencode(
example_query_params)
if example_body:
endpoint["example"][
@ -631,12 +632,12 @@ class MatrixUnits(Units):
body_tables = req_body_tables[1:]
endpoint_data['req_body_tables'].extend(body_tables)
except Exception, e:
except Exception as e:
e2 = Exception(
"Error decoding body of API endpoint %s %s: %s" %
(endpoint_data["method"], endpoint_data["path"], e)
)
raise e2, None, sys.exc_info()[2]
raise e2.with_traceback(sys.exc_info()[2])
def load_swagger_apis(self):
@ -737,12 +738,12 @@ class MatrixUnits(Units):
if filename != event_name:
examples[event_name] = examples.get(event_name, [])
examples[event_name].append(example)
except Exception, e:
except Exception as e:
e2 = Exception("Error reading event example "+filepath+": "+
str(e))
# throw the new exception with the old stack trace, so that
# we don't lose information about where the error occurred.
raise e2, None, sys.exc_info()[2]
raise e2.with_traceback(sys.exc_info()[2])
return examples
@ -756,12 +757,12 @@ class MatrixUnits(Units):
filepath = os.path.join(path, filename)
try:
schemata[filename] = self.read_event_schema(filepath)
except Exception, e:
except Exception as e:
e2 = Exception("Error reading event schema "+filepath+": "+
str(e))
# throw the new exception with the old stack trace, so that
# we don't lose information about where the error occurred.
raise e2, None, sys.exc_info()[2]
raise e2.with_traceback(sys.exc_info()[2])
return schemata
@ -857,12 +858,42 @@ class MatrixUnits(Units):
path = os.path.join(CHANGELOG_DIR, f)
name = f[:-4]
# If there's a directory with the same name, we'll try to generate
# a towncrier changelog and prepend it to the general changelog.
tc_path = os.path.join(CHANGELOG_DIR, name)
tc_lines = []
if os.path.isdir(tc_path):
logger.info("Generating towncrier changelog for: %s" % name)
p = subprocess.Popen(
['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'],
cwd=tc_path,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
stdout, stderr = p.communicate()
if p.returncode != 0:
# Something broke - dump as much information as we can
logger.error("Towncrier exited with code %s" % p.returncode)
logger.error(stdout.decode('UTF-8'))
logger.error(stderr.decode('UTF-8'))
raw_log = ""
else:
raw_log = stdout.decode('UTF-8')
# This is a bit of a hack, but it does mean that the log at least gets *something*
# to tell us it broke
if not raw_log.startswith("Unreleased Changes"):
logger.error("Towncrier appears to have failed to generate a changelog")
logger.error(raw_log)
raw_log = ""
tc_lines = raw_log.splitlines()
title_part = None
changelog_lines = []
with open(path, "r") as f:
lines = f.readlines()
prev_line = None
for line in lines:
for line in (tc_lines + lines):
if prev_line is None:
prev_line = line
continue
@ -878,7 +909,10 @@ class MatrixUnits(Units):
# then bail out.
changelog_lines.pop()
break
changelog_lines.append(" " + line)
# Don't generate subheadings (we'll keep the title though)
if re.match("^[-]{3,}$", line.strip()):
continue
changelog_lines.append(" " + line + '\n')
changelogs[name] = "".join(changelog_lines)
return changelogs
@ -897,7 +931,7 @@ class MatrixUnits(Units):
['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
stderr=null,
cwd=cwd,
).strip()
).strip().decode('UTF-8')
except subprocess.CalledProcessError:
git_branch = ""
try:
@ -905,7 +939,7 @@ class MatrixUnits(Units):
['git', 'describe', '--exact-match'],
stderr=null,
cwd=cwd,
).strip()
).strip().decode('UTF-8')
git_tag = "tag=" + git_tag
except subprocess.CalledProcessError:
git_tag = ""
@ -914,7 +948,7 @@ class MatrixUnits(Units):
['git', 'rev-parse', '--short', 'HEAD'],
stderr=null,
cwd=cwd,
).strip()
).strip().decode('UTF-8')
except subprocess.CalledProcessError:
git_commit = ""
try:
@ -923,7 +957,7 @@ class MatrixUnits(Units):
['git', 'describe', '--dirty=' + dirty_string, "--all"],
stderr=null,
cwd=cwd,
).strip().endswith(dirty_string)
).strip().decode('UTF-8').endswith(dirty_string)
git_dirty = "dirty" if is_dirty else ""
except subprocess.CalledProcessError:
git_dirty = ""
@ -934,7 +968,7 @@ class MatrixUnits(Units):
s for s in
(git_branch, git_tag, git_commit, git_dirty,)
if s
).encode("ascii")
).encode("ascii").decode('ascii')
return {
"string": git_version,
"revision": git_commit

7
scripts/test-and-build.sh
View file

@ -4,8 +4,13 @@ set -ex
cd `dirname $0`/..
virtualenv env
virtualenv -p python3 env
. env/bin/activate
# Print out the python versions for debugging purposes
python --version
pip --version
pip install -r scripts/requirements.txt
# do sanity checks on the examples and swagger