Move templating into scripts dir
There's no real need for this to be at the top level.
This commit is contained in:
Richard van der Hoff
parent
d9285cf5b5
commit
a38d4fc68e
18 changed files with 17 additions and 18 deletions
285
scripts/templating/build.py
Executable file
285
scripts/templating/build.py
Executable file
|
|
@ -0,0 +1,285 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2016 OpenMarket Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Batesian: A simple templating system using Jinja.
|
||||
|
||||
Architecture
|
||||
============
|
||||
|
||||
INPUT FILE --------+
|
||||
+-------+ +----------+ |
|
||||
| units |-+ | sections |-+ V
|
||||
+-------+ |-+ == used to create ==> +----------- | == provides vars to ==> Jinja
|
||||
+-------+ | +----------+ |
|
||||
+--------+ V
|
||||
RAW DATA (e.g. json) Blobs of text OUTPUT FILE
|
||||
|
||||
Units
|
||||
=====
|
||||
Units are random bits of unprocessed data, e.g. schema JSON files. Anything can
|
||||
be done to them, from processing it with Jinja to arbitrary python processing.
|
||||
They are typically dicts.
|
||||
|
||||
Sections
|
||||
========
|
||||
Sections are strings, typically short segments of RST. They will be dropped in
|
||||
to the provided input file based on their section key name (template var)
|
||||
They typically use a combination of templates + units to construct bits of RST.
|
||||
|
||||
Input File
|
||||
==========
|
||||
The input file is a text file which is passed through Jinja along with the
|
||||
section keys as template variables.
|
||||
|
||||
Processing
|
||||
==========
|
||||
- Execute all unit functions to load units into memory and process them.
|
||||
- Execute all section functions (which can now be done because the units exist)
|
||||
- Process the input file through Jinja, giving it the sections as template vars.
|
||||
"""
|
||||
from batesian import AccessKeyStore
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, meta
|
||||
from argparse import ArgumentParser, FileType
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from textwrap import TextWrapper
|
||||
|
||||
from matrix_templates.units import TypeTableRow
|
||||
|
||||
|
||||
def create_from_template(template, sections):
|
||||
return template.render(sections)
|
||||
|
||||
def check_unaccessed(name, store):
|
||||
unaccessed_keys = store.get_unaccessed_set()
|
||||
if len(unaccessed_keys) > 0:
|
||||
log("Found %s unused %s keys." % (len(unaccessed_keys), name))
|
||||
log(unaccessed_keys)
|
||||
|
||||
def main(input_module, files=None, out_dir=None, verbose=False, substitutions={}):
|
||||
if out_dir and not os.path.exists(out_dir):
|
||||
os.makedirs(out_dir)
|
||||
|
||||
in_mod = importlib.import_module(input_module)
|
||||
|
||||
# add a template filter to produce pretty pretty JSON
|
||||
def jsonify(input, indent=None, pre_whitespace=0):
|
||||
code = json.dumps(input, indent=indent, sort_keys=True)
|
||||
if pre_whitespace:
|
||||
code = code.replace("\n", ("\n" +" "*pre_whitespace))
|
||||
|
||||
return code
|
||||
|
||||
def indent_block(input, indent):
|
||||
return input.replace("\n", ("\n" + " "*indent))
|
||||
|
||||
def indent(input, indent):
|
||||
return " "*indent + input
|
||||
|
||||
def wrap(input, wrap=80, initial_indent=""):
|
||||
if len(input) == 0:
|
||||
return initial_indent
|
||||
# TextWrapper collapses newlines into single spaces; we do our own
|
||||
# splitting on newlines to prevent this, so that newlines can actually
|
||||
# be intentionally inserted in text.
|
||||
input_lines = input.split('\n\n')
|
||||
wrapper = TextWrapper(initial_indent=initial_indent, width=wrap)
|
||||
output_lines = [wrapper.fill(line) for line in input_lines]
|
||||
|
||||
for i in range(len(output_lines)):
|
||||
line = output_lines[i]
|
||||
in_bullet = line.startswith("- ")
|
||||
if in_bullet:
|
||||
output_lines[i] = line.replace("\n", "\n " + initial_indent)
|
||||
|
||||
return '\n\n'.join(output_lines)
|
||||
|
||||
def fieldwidths(input, keys, defaults=[], default_width=15):
|
||||
"""
|
||||
A template filter to help in the generation of tables.
|
||||
|
||||
Given a list of rows, returns a list giving the maximum length of the
|
||||
values in each column.
|
||||
|
||||
:param list[TypeTableRow|dict[str,str]] input:
|
||||
a list of rows
|
||||
:param list[str] keys: the keys corresponding to the table columns
|
||||
:param list[int] defaults: for each column, the default column width.
|
||||
:param int default_width: if ``defaults`` is shorter than ``keys``, this
|
||||
will be used as a fallback
|
||||
"""
|
||||
def getrowattribute(row, k):
|
||||
# the row may be a dict (particularly the title row, which is
|
||||
# generated by the template
|
||||
if not isinstance(row, TypeTableRow):
|
||||
return row[k]
|
||||
return getattr(row, k)
|
||||
|
||||
def colwidth(key, default):
|
||||
rowwidths = (len(getrowattribute(row, key)) for row in input)
|
||||
return reduce(max, rowwidths,
|
||||
default if default is not None else default_width)
|
||||
|
||||
results = map(colwidth, keys, defaults)
|
||||
return results
|
||||
|
||||
# make Jinja aware of the templates and filters
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(in_mod.exports["templates"]),
|
||||
undefined=StrictUndefined
|
||||
)
|
||||
env.filters["jsonify"] = jsonify
|
||||
env.filters["indent"] = indent
|
||||
env.filters["indent_block"] = indent_block
|
||||
env.filters["wrap"] = wrap
|
||||
env.filters["fieldwidths"] = fieldwidths
|
||||
|
||||
# load up and parse the lowest single units possible: we don't know or care
|
||||
# which spec section will use it, we just need it there in memory for when
|
||||
# they want it.
|
||||
units = AccessKeyStore(
|
||||
existing_data=in_mod.exports["units"](
|
||||
debug=verbose,
|
||||
substitutions=substitutions,
|
||||
).get_units()
|
||||
)
|
||||
|
||||
# use the units to create RST sections
|
||||
sections = in_mod.exports["sections"](env, units, debug=verbose).get_sections()
|
||||
|
||||
# print out valid section keys if no file supplied
|
||||
if not files:
|
||||
print "\nValid template variables:"
|
||||
for key in sections.keys():
|
||||
sec_text = "" if (len(sections[key]) > 75) else (
|
||||
"(Value: '%s')" % sections[key]
|
||||
)
|
||||
sec_info = "%s characters" % len(sections[key])
|
||||
if sections[key].count("\n") > 0:
|
||||
sec_info += ", %s lines" % sections[key].count("\n")
|
||||
print " %s" % key
|
||||
print " %s %s" % (sec_info, sec_text)
|
||||
return
|
||||
|
||||
# check the input files and substitute in sections where required
|
||||
for input_filename in files:
|
||||
output_filename = os.path.join(out_dir,
|
||||
os.path.basename(input_filename))
|
||||
process_file(env, sections, input_filename, output_filename)
|
||||
|
||||
check_unaccessed("units", units)
|
||||
|
||||
def process_file(env, sections, filename, output_filename):
|
||||
log("Parsing input template: %s" % filename)
|
||||
|
||||
with open(filename, "r") as file_stream:
|
||||
temp_str = file_stream.read().decode("utf-8")
|
||||
|
||||
# do sanity checking on the template to make sure they aren't reffing things
|
||||
# which will never be replaced with a section.
|
||||
ast = env.parse(temp_str)
|
||||
template_vars = meta.find_undeclared_variables(ast)
|
||||
unused_vars = [var for var in template_vars if var not in sections]
|
||||
if len(unused_vars) > 0:
|
||||
raise Exception(
|
||||
"You have {{ variables }} which are not found in sections: %s" %
|
||||
(unused_vars,)
|
||||
)
|
||||
# process the template
|
||||
temp = Template(temp_str)
|
||||
output = create_from_template(temp, sections)
|
||||
|
||||
# Do these substitutions outside of the ordinary templating system because
|
||||
# we want them to apply to things like the underlying swagger used to
|
||||
# generate the templates, not just the top-level sections.
|
||||
for old, new in substitutions.items():
|
||||
output = output.replace(old, new)
|
||||
|
||||
with open(output_filename, "w") as f:
|
||||
f.write(output.encode("utf-8"))
|
||||
log("Output file for: %s" % output_filename)
|
||||
|
||||
|
||||
def log(line):
|
||||
print "batesian: %s" % line
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(
|
||||
"Processes a file (typically .rst) through Jinja to replace templated "+
|
||||
"areas with section information from the provided input module. For a "+
|
||||
"list of possible template variables, add --show-template-vars."
|
||||
)
|
||||
parser.add_argument(
|
||||
"files", nargs="+",
|
||||
help="The input files to process. These will be passed through Jinja "+
|
||||
"then output under the same name to the output directory."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input", "-i",
|
||||
help="The python module (not file) which contains the sections/units "+
|
||||
"classes. This module must have an 'exports' dict which has "+
|
||||
"{ 'units': UnitClass, 'sections': SectionClass, "+
|
||||
"'templates': 'template/dir' }"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out-directory", "-o", help="The directory to output the file to."+
|
||||
" Default: /out",
|
||||
default="out"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--show-template-vars", "-s", action="store_true",
|
||||
help="Show a list of all possible variables (sections) you can use in"+
|
||||
" the input file."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", "-v", action="store_true",
|
||||
help="Turn on verbose mode."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--substitution", action="append",
|
||||
help="Substitutions to apply to the generated output, of form NEEDLE=REPLACEMENT.",
|
||||
default=[],
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
if not args.input:
|
||||
raise Exception("Missing [i]nput python module.")
|
||||
|
||||
if (args.show_template_vars):
|
||||
main(args.input, verbose=args.verbose)
|
||||
sys.exit(0)
|
||||
|
||||
substitutions = {}
|
||||
for substitution in args.substitution:
|
||||
parts = substitution.split("=", 1)
|
||||
if len(parts) != 2:
|
||||
raise Exception("Invalid substitution")
|
||||
substitutions[parts[0]] = parts[1]
|
||||
|
||||
main(
|
||||
args.input, files=args.files, out_dir=args.out_directory,
|
||||
substitutions=substitutions, verbose=args.verbose
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue