From 064a2c91721a285cf03dcd0505e3c01a8af39256 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 14:59:57 +0100 Subject: [PATCH 1/3] Use argparse and log functions for gendoc.py gendoc.py has become more complex such that we actually want to pass things to it like `--verbose`, `--nodelete`, `--target`, so use `argparse` to do this like we have `build.py`. Pass through `-v` flags to `build.py`. --- scripts/gendoc.py | 91 +++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 364b8e65..d2b0d75e 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -1,5 +1,6 @@ #! /usr/bin/env python +from argparse import ArgumentParser from docutils.core import publish_file import copy import fileinput @@ -17,6 +18,7 @@ stylesheets = { "stylesheet_path": ["basic.css", "nature.css", "codehighlight.css"] } +VERBOSE = False """ Read a RST file and replace titles with a different title level if required. @@ -64,8 +66,8 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): if file_offset is None: file_offset = line_title_level if file_offset != 0: - print (" WARNING: %s starts with a title style of '%s' but '%s' " + - "is preferable.") % (filename, line_title_style, title_styles[0]) + logv((" WARNING: %s starts with a title style of '%s' but '%s' " + + "is preferable.") % (filename, line_title_style, title_styles[0])) # Sanity checks: Make sure that this file is obeying the title levels # specified and bail if it isn't. @@ -101,12 +103,11 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): continue # Adjusting line levels - # print ( - # "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % - # (filename, line_title_style, - # title_styles[adjusted_level], - # file_offset, title_level) - # ) + logv( + "File: %s Adjusting %s to %s because file_offset=%s title_offset=%s" % + (filename, line_title_style, title_styles[adjusted_level], + file_offset, title_level) + ) rst_lines.append(line.replace( line_title_style, title_styles[adjusted_level] @@ -118,7 +119,7 @@ def load_with_adjusted_titles(filename, file_stream, title_level, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, basestring): - print "%s %s" % (">" * (1 + title_level), file_info) + log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None if adjust_titles: @@ -244,15 +245,21 @@ def run_through_template(input): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: - print subprocess.check_output( - [ - 'python', 'build.py', "-v", - "-i", "matrix_templates", - "-o", "../scripts/tmp", - "../scripts/"+input - ], - stderr=out, - cwd="../templating", + args = [ + 'python', 'build.py', + "-i", "matrix_templates", + "-o", "../scripts/tmp", + "../scripts/"+input + ] + if VERBOSE: + args.insert(2, "-v") + log(" ==== build.py output ==== ") + log( + subprocess.check_output( + args, + stderr=out, + cwd="../templating" + ) ) except subprocess.CalledProcessError as e: with open(tmpfile, 'r') as f: @@ -347,6 +354,13 @@ def get_build_target(targets_listing, target_name): build_target["files"] = resolved_files return build_target +def log(line): + print "gendoc: %s" % line + +def logv(line): + if VERBOSE: + print "gendoc:V: %s" % line + def prepare_env(): try: @@ -363,9 +377,9 @@ def cleanup_env(): shutil.rmtree("./tmp") -def main(target_name): +def main(target_name, keep_intermediates): prepare_env() - print "Building spec [target=%s]" % target_name + log("Building spec [target=%s]" % target_name) target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/templated_spec.rst") run_through_template("tmp/templated_spec.rst") @@ -377,22 +391,29 @@ def main(target_name): run_through_template("tmp/howto.rst") rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") - if "--nodelete" not in sys.argv: + if not keep_intermediates: cleanup_env() if __name__ == '__main__': - if len(sys.argv) > 1 and sys.argv[1:] != ["--nodelete"]: - # we accept almost no args, so they don't know what they're doing! - print "gendoc.py - Generate the Matrix specification as HTML." - print "Usage:" - print " python gendoc.py [--nodelete]" - print "" - print "The specification can then be found in the gen/ folder." - print ("If --nodelete was specified, intermediate files will be " - "present in the tmp/ folder.") - print "" - print "Requirements:" - print " - This script requires Jinja2 and rst2html (docutils)." - sys.exit(0) - main("main") + parser = ArgumentParser( + "gendoc.py - Generate the Matrix specification as HTML to the gen/ folder." + ) + parser.add_argument( + "--nodelete", "-n", action="store_true", + help="Do not delete intermediate files. They will be found in tmp/" + ) + parser.add_argument( + "--target", "-t", default="main", + help="Specify the build target to build from specification/targets.yaml" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Turn on verbose mode." + ) + args = parser.parse_args() + if not args.target: + parser.print_help() + sys.exit(1) + VERBOSE = args.verbose + main(args.target, args.nodelete) From f1adad5fb3b2c6f03d1b635880726a661ba7bb03 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 15:10:55 +0100 Subject: [PATCH 2/3] Add more logging with file prefixes This makes the handoff between gendoc and batesian clearer in the logs. --- scripts/gendoc.py | 11 +++++------ templating/batesian/sections.py | 2 +- templating/batesian/units.py | 2 +- templating/build.py | 14 ++++++++------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index d2b0d75e..611d0d10 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -253,13 +253,12 @@ def run_through_template(input): ] if VERBOSE: args.insert(2, "-v") + log("EXEC: %s" % " ".join(args)) log(" ==== build.py output ==== ") - log( - subprocess.check_output( - args, - stderr=out, - cwd="../templating" - ) + print subprocess.check_output( + args, + stderr=out, + cwd="../templating" ) except subprocess.CalledProcessError as e: with open(tmpfile, 'r') as f: diff --git a/templating/batesian/sections.py b/templating/batesian/sections.py index 31849389..c18c9a5f 100644 --- a/templating/batesian/sections.py +++ b/templating/batesian/sections.py @@ -16,7 +16,7 @@ class Sections(object): def log(self, text): if self.debug: - print text + print "batesian:sections: %s" % text def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/batesian/units.py b/templating/batesian/units.py index c20a2d1f..993f253d 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,7 @@ class Units(object): def log(self, text): if self.debug: - print text + print "batesian:units: %s" % text def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/build.py b/templating/build.py index ac3d2491..013248f4 100755 --- a/templating/build.py +++ b/templating/build.py @@ -52,8 +52,8 @@ def create_from_template(template, sections): def check_unaccessed(name, store): unaccessed_keys = store.get_unaccessed_set() if len(unaccessed_keys) > 0: - print "Found %s unused %s keys." % (len(unaccessed_keys), name) - print unaccessed_keys + log("Found %s unused %s keys." % (len(unaccessed_keys), name)) + log(unaccessed_keys) def main(input_module, file_stream=None, out_dir=None, verbose=False): if out_dir and not os.path.exists(out_dir): @@ -121,17 +121,19 @@ def main(input_module, file_stream=None, out_dir=None, verbose=False): return # check the input files and substitute in sections where required - print "Parsing input template: %s" % file_stream.name + log("Parsing input template: %s" % file_stream.name) temp = Template(file_stream.read()) - print "Creating output for: %s" % file_stream.name + log("Creating output for: %s" % file_stream.name) output = create_from_template(temp, sections) with open( os.path.join(out_dir, os.path.basename(file_stream.name)), "w" ) as f: f.write(output) - print "Output file for: %s" % file_stream.name + log("Output file for: %s" % file_stream.name) check_unaccessed("units", units) +def log(line): + print "batesian: %s" % line if __name__ == '__main__': parser = ArgumentParser( @@ -175,7 +177,7 @@ if __name__ == '__main__': sys.exit(0) if not args.file: - print "No file supplied." + log("No file supplied.") parser.print_help() sys.exit(1) From 6afdfc0771265ded3b7e0752c7ee438b09f076cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 23 Sep 2015 15:36:13 +0100 Subject: [PATCH 3/3] Add more logging and make logging context clearer This is now actually useful if you want to debug why your swagger YAML isn't producing a table you think it should be. --- scripts/gendoc.py | 8 ++++---- templating/batesian/units.py | 6 +++++- templating/matrix_templates/units.py | 28 +++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 611d0d10..1655d6f0 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -241,7 +241,7 @@ def rst2html(i, o): ) -def run_through_template(input): +def run_through_template(input, set_verbose): tmpfile = './tmp/output' try: with open(tmpfile, 'w') as out: @@ -251,7 +251,7 @@ def run_through_template(input): "-o", "../scripts/tmp", "../scripts/"+input ] - if VERBOSE: + if set_verbose: args.insert(2, "-v") log("EXEC: %s" % " ".join(args)) log(" ==== build.py output ==== ") @@ -381,13 +381,13 @@ def main(target_name, keep_intermediates): log("Building spec [target=%s]" % target_name) target = get_build_target("../specification/targets.yaml", target_name) build_spec(target=target, out_filename="tmp/templated_spec.rst") - run_through_template("tmp/templated_spec.rst") + run_through_template("tmp/templated_spec.rst", VERBOSE) fix_relative_titles( target=target, filename="tmp/templated_spec.rst", out_filename="tmp/full_spec.rst" ) shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst") - run_through_template("tmp/howto.rst") + run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this rst2html("tmp/full_spec.rst", "gen/specification.html") rst2html("tmp/howto.rst", "gen/howtos.html") if not keep_intermediates: diff --git a/templating/batesian/units.py b/templating/batesian/units.py index 993f253d..144cf245 100644 --- a/templating/batesian/units.py +++ b/templating/batesian/units.py @@ -22,7 +22,11 @@ class Units(object): def log(self, text): if self.debug: - print "batesian:units: %s" % text + func_name = "" + trace = inspect.stack() + if len(trace) > 1 and len(trace[1]) > 2: + func_name = trace[1][3] + ":" + print "batesian:units:%s %s" % (func_name, text) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py index a5c6a815..50fa784e 100644 --- a/templating/matrix_templates/units.py +++ b/templating/matrix_templates/units.py @@ -1,4 +1,12 @@ -"""Contains all the units for the spec.""" +""" +Contains all the units for the spec. + +This file loads swagger and JSON schema files and parses out the useful bits +and returns them as Units for use in Batesian. + +For the actual conversion of data -> RST (including templates), see the sections +file instead. +""" from batesian.units import Units import inspect import json @@ -134,7 +142,7 @@ class MatrixUnits(Units): "good_response": "" } } - self.log(".o.O.o. Endpoint: %s %s" % (method, path)) + self.log(" ------- Endpoint: %s %s ------- " % (method, path)) for param in single_api.get("parameters", []): # description desc = param.get("description", "") @@ -183,6 +191,9 @@ class MatrixUnits(Units): "desc": json_body[key]["description"] }) # endfor[param] + for row in endpoint["req_params"]: + self.log("Request parameter: %s" % row) + # group params by location to ease templating endpoint["req_param_by_loc"] = { # path: [...], query: [...], body: [...] @@ -240,6 +251,7 @@ class MatrixUnits(Units): # add response params if this API has any. if good_response: + self.log("Found a 200 response for this API") res_type = Units.prop(good_response, "schema/type") if res_type and res_type not in ["object", "array"]: # response is a raw string or something like that @@ -278,6 +290,16 @@ class MatrixUnits(Units): }] }) + for response_table in endpoint["res_tables"]: + self.log("Response: %s" % response_table["title"]) + for r in response_table["rows"]: + self.log("Row: %s" % r) + if len(endpoint["res_tables"]) == 0: + self.log( + "This API appears to have no response table. Are you " + + "sure this API returns no parameters?" + ) + endpoints.append(endpoint) aliases = single_api.get("x-alias", None) @@ -475,7 +497,7 @@ class MatrixUnits(Units): if re.match("^v[0-9\.]+$", word): version = word[1:] # strip the 'v' - self.log("Version: %s Title part: %s Changelog lines: %s" % ( + self.log("Version: %s Title part: %s Changelog line count: %s" % ( version, title_part, len(changelog_lines) )) if not version or len(changelog_lines) == 0: