wholetrans/docs-matrix-spec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #1626 from matrix-org/rav/encoding

Browse source 
Set the encoding to UTF-8 on all file I/O
This commit is contained in:
Richard van der Hoff 2018-08-30 18:11:38 +01:00 • committed by GitHub
commit f649d04ae1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

21
scripts/gendoc.py
View file

@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
# string are file paths to RST blobs
if isinstance(file_info, str):
log("%s %s" % (">" * (1 + title_level), file_info))
with open(os.path.join(spec_dir, file_info), "r") as f:
with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f:
rst = None
if adjust_titles:
rst = load_with_adjusted_titles(
@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles):
def build_spec(target, out_filename):
log("Building templated file %s" % out_filename)
with open(out_filename, "wb") as outfile:
with open(out_filename, "w", encoding="utf-8") as outfile:
for file_info in target["files"]:
section = get_rst(
file_info=file_info,
@ -195,7 +195,7 @@ def build_spec(target, out_filename):
spec_dir=spec_dir,
adjust_titles=True
)
outfile.write(section.encode('UTF-8'))
outfile.write(section)
"""
@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename):
"^[" + re.escape("".join(title_styles)) + "]{3,}$"
)
current_title_style = None
with open(filename, "r") as infile:
with open(out_filename, "w") as outfile:
with open(filename, "r", encoding="utf-8") as infile:
with open(out_filename, "w", encoding="utf-8") as outfile:
for line in infile.readlines():
if not relative_title_matcher.match(line):
if title_matcher.match(line):
@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename):
def rst2html(i, o, stylesheets):
log("rst2html %s -> %s" % (i, o))
with open(i, "r") as in_file:
with open(o, "w") as out_file:
with open(i, "r", encoding="utf-8") as in_file:
with open(o, "w", encoding="utf-8") as out_file:
publish_file(
source=in_file,
destination=out_file,
@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets):
def addAnchors(path):
log("add anchors %s" % path)
with open(path, "rb") as f:
with open(path, "r", encoding="utf-8") as f:
lines = f.readlines()
replacement = r'<p><a class="anchor" id="\2"></a></p>\n\1'
with open(path, "wb") as f:
with open(path, "w", encoding="utf-8") as f:
for line in lines:
line = line.decode("UTF-8")
line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
line = re.sub(r'(<div class="section" id="(.*?)">)', replacement, line.rstrip())
f.write((line + "\n").encode('UTF-8'))
f.write(line + "\n")
def run_through_template(input_files, set_verbose, substitutions):

18
scripts/templating/matrix_templates/units.py
View file

@ -125,7 +125,7 @@ def resolve_references(path, schema):
if '$ref' in schema:
value = schema['$ref']
path = os.path.join(os.path.dirname(path), value)
with open(path) as f:
with open(path, encoding="utf-8") as f:
ref = yaml.load(f, OrderedLoader)
result = resolve_references(path, ref)
del schema['$ref']
@ -664,11 +664,11 @@ class MatrixUnits(Units):
continue
filepath = os.path.join(path, filename)
logger.info("Reading swagger API: %s" % filepath)
with open(filepath, "r") as f:
with open(filepath, "r", encoding="utf-8") as f:
# strip .yaml
group_name = filename[:-5].replace("-", "_")
group_name = "%s_%s" % (group_name, suffix)
api = yaml.load(f.read(), OrderedLoader)
api = yaml.load(f, OrderedLoader)
api = resolve_references(filepath, api)
api["__meta"] = self._load_swagger_meta(
api, group_name
@ -698,11 +698,11 @@ class MatrixUnits(Units):
continue
filepath = os.path.join(path, filename)
logger.info("Reading swagger definition: %s" % filepath)
with open(filepath, "r") as f:
with open(filepath, "r", encoding="utf-8") as f:
# strip .yaml
group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5])
group_name = "%s_%s" % (prefix, group_name)
definition = yaml.load(f.read(), OrderedLoader)
definition = yaml.load(f, OrderedLoader)
definition = resolve_references(filepath, definition)
if 'type' not in definition:
continue
@ -741,7 +741,7 @@ class MatrixUnits(Units):
event_type = filename[:-5] # strip the ".yaml"
logger.info("Reading event schema: %s" % filepath)
with open(filepath) as f:
with open(filepath, encoding="utf-8") as f:
event_schema = yaml.load(f, OrderedLoader)
schema_info = process_data_type(
@ -793,7 +793,7 @@ class MatrixUnits(Units):
filepath = os.path.join(path, filename)
logger.info("Reading event example: %s" % filepath)
try:
with open(filepath, "r") as f:
with open(filepath, "r", encoding="utf-8") as f:
example = json.load(f)
examples[filename] = examples.get(filename, [])
examples[filename].append(example)
@ -831,7 +831,7 @@ class MatrixUnits(Units):
def read_event_schema(self, filepath):
logger.info("Reading %s" % filepath)
with open(filepath, "r") as f:
with open(filepath, "r", encoding="utf-8") as f:
json_schema = yaml.load(f, OrderedLoader)
schema = {
@ -943,7 +943,7 @@ class MatrixUnits(Units):
title_part = None
changelog_lines = []
with open(path, "r") as f:
with open(path, "r", encoding="utf-8") as f:
lines = f.readlines()
prev_line = None
for line in (tc_lines + lines):