Merge pull request #29 from edx/bmedx/docgen

Add the ability to generate human readable reports in RST from code-annotations generated YAML files

.annotations_sample

@@ -1,7 +1,11 @@
-source_path: ../
+source_path: ./
 report_path: reports
 safelist_path: .annotation_safe_list.yml
 coverage_target: 50.0
+report_template_dir: code_annotations/report_templates/
+rendered_report_dir: code_annotations/reports/
+rendered_report_file_extension: .rst
+rendered_report_source_link_prefix: https://github.com/edx/edx-platform/tree/master/
 annotations:
     ".. no_pii:":
     ".. ignored:":

code_annotations/__init__.py

@@ -4,4 +4,4 @@
 
 from __future__ import absolute_import, unicode_literals
 
-__version__ = '0.2.4'
+__version__ = '0.3'

code_annotations/base.py

@@ -21,7 +21,7 @@ class AnnotationConfig(object):
     Configuration shared among all Code Annotations commands.
     """
 
-    def __init__(self, config_file_path, report_path_override, verbosity, source_path_override=None):
+    def __init__(self, config_file_path, report_path_override=None, verbosity=1, source_path_override=None):
         """
          Initialize AnnotationConfig.
 
@@ -41,7 +41,7 @@ def __init__(self, config_file_path, report_path_override, verbosity, source_pat
         self.echo = VerboseEcho()
 
         with open(config_file_path) as config_file:
-            raw_config = yaml.load(config_file)
+            raw_config = yaml.safe_load(config_file)
 
         self._check_raw_config_keys(raw_config)
 
@@ -58,6 +58,11 @@ def __init__(self, config_file_path, report_path_override, verbosity, source_pat
         self.echo("Configured for source path: {}".format(self.source_path))
 
         self._configure_coverage(raw_config.get('coverage_target', None))
+        self.report_template_dir = raw_config.get('report_template_dir')
+        self.rendered_report_dir = raw_config.get('rendered_report_dir')
+        self.rendered_report_file_extension = raw_config.get('rendered_report_file_extension')
+        self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix')
+
         self._configure_annotations(raw_config)
         self._configure_extensions()
 
@@ -599,6 +604,6 @@ def report(self, all_results, report_prefix=''):
                 raise
 
         with open(report_filename, 'w+') as report_file:
-            yaml.dump(formatted_results, report_file, default_flow_style=False)
+            yaml.safe_dump(formatted_results, report_file, default_flow_style=False)
 
         return report_filename

code_annotations/cli.py

@@ -9,6 +9,7 @@
 from code_annotations.base import AnnotationConfig, ConfigurationException
 from code_annotations.find_django import DjangoSearch
 from code_annotations.find_static import StaticSearch
+from code_annotations.generate_docs import ReportRenderer
 from code_annotations.helpers import fail
 
 
@@ -171,3 +172,46 @@ def static_find_annotations(config_file, source_path, report_path, verbosity, li
     except Exception as exc:  # pylint: disable=broad-except
         click.echo(traceback.print_exc())
         fail(str(exc))
+
+
+@entry_point.command("generate_docs")
+@click.option(
+    '--config_file',
+    default='.annotations',
+    help='Path to the configuration file',
+    type=click.Path(exists=True, dir_okay=False)
+)
+@click.option('-v', '--verbosity', count=True, help='Verbosity level (-v through -vvv)')
+@click.argument("report_files", type=click.File('r'), nargs=-1)
+def generate_docs(
+        config_file,
+        verbosity,
+        report_files
+):
+    """
+    Generate documentation from a code annotations report.
+    """
+    start_time = datetime.datetime.now()
+
+    try:
+        config = AnnotationConfig(config_file, verbosity)
+
+        for key in (
+                'report_template_dir',
+                'rendered_report_dir',
+                'rendered_report_file_extension',
+                'rendered_report_source_link_prefix'
+        ):
+            if not getattr(config, key):
+                raise ConfigurationException("No {key} key in {config_file}".format(key=key, config_file=config_file))
+
+        config.echo("Rendering the following reports: \n{}".format("\n".join([r.name for r in report_files])))
+
+        renderer = ReportRenderer(config, report_files)
+        renderer.render()
+
+        elapsed = datetime.datetime.now() - start_time
+        click.echo("Report rendered in {} seconds.".format(elapsed.total_seconds()))
+    except Exception as exc:  # pylint: disable=broad-except
+        click.echo(traceback.print_exc())
+        fail(str(exc))

code_annotations/find_django.py

@@ -7,13 +7,15 @@
 import re
 import sys
 
+# pylint: disable=ungrouped-imports
 import django
+import yaml
 from django.apps import apps
 from django.db import models
 from six import text_type
 
 from code_annotations.base import BaseSearch
-from code_annotations.helpers import fail, get_annotation_regex, yaml_ordered_dump, yaml_ordered_load
+from code_annotations.helpers import fail, get_annotation_regex
 
 DEFAULT_SAFELIST_FILE_PATH = '.annotation_safe_list.yml'
 
@@ -75,7 +77,7 @@ def seed_safelist(self):
 
 """
             safelist_file.write(safelist_comment.lstrip())
-            yaml_ordered_dump(safelist_data, stream=safelist_file, default_flow_style=False)
+            yaml.safe_dump(safelist_data, stream=safelist_file, default_flow_style=False)
 
         self.echo('Successfully created safelist file "{}".'.format(self.config.safelist_path), fg='red')
         self.echo('Now, you need to:', fg='red')
@@ -175,7 +177,7 @@ def _read_safelist(self):
         if os.path.exists(self.config.safelist_path):
             self.echo('Found safelist at {}. Reading.\n'.format(self.config.safelist_path))
             with open(self.config.safelist_path) as safelist_file:
-                safelisted_models = yaml_ordered_load(safelist_file)
+                safelisted_models = yaml.safe_load(safelist_file)
             self._increment_count('safelisted', len(safelisted_models))
 
             if safelisted_models:

code_annotations/generate_docs.py

@@ -0,0 +1,145 @@
+"""
+Contains functionality for turning YAML reports into human-readable documentation.
+"""
+
+import collections
+import datetime
+import os
+
+import jinja2
+import yaml
+from slugify import slugify
+
+
+class ReportRenderer(object):
+    """
+    Generates human readable documentation from YAML reports.
+    """
+
+    def __init__(self, config, report_files):
+        """
+        Initialize a ReportRenderer.
+
+        Args:
+            config: An AnnotationConfig object
+            report_files: A list of files to combine and report on
+        """
+        self.config = config
+        self.echo = self.config.echo
+        self.report_files = report_files
+        self.create_time = datetime.datetime.now().isoformat()
+
+        self.full_report = self._aggregate_reports()
+
+        self.jinja_environment = jinja2.Environment(
+            autoescape=False,
+            loader=jinja2.FileSystemLoader(self.config.report_template_dir),
+            lstrip_blocks=True,
+            trim_blocks=True
+        )
+        self.top_level_template = self.jinja_environment.get_template('annotation_list.tpl')
+        self.all_choices = []
+        self.group_mapping = {}
+
+        for token in self.config.choices:
+            self.all_choices.extend(self.config.choices[token])
+
+        for group_name in self.config.groups:
+            for token in self.config.groups[group_name]:
+                self.group_mapping[token] = group_name
+
+    def _add_report_file_to_full_report(self, report_file, report):
+        loaded_report = yaml.safe_load(report_file)
+
+        for filename in loaded_report:
+            if filename in report:
+                for loaded_annotation in loaded_report[filename]:
+                    found = False
+                    for report_annotation in report[filename]:
+                        index_keys = ('line_number', 'annotation_token', 'annotation_data')
+
+                        if all([loaded_annotation[k] == report_annotation[k] for k in index_keys]):
+                            report_annotation.update(loaded_annotation)
+                            found = True
+                            break
+
+                    if not found:
+                        report[filename].append(loaded_annotation)
+            else:
+                report[filename] = loaded_report[filename]
+
+    def _aggregate_reports(self):
+        """
+        Combine all of the given report files into a single report object.
+        """
+        report = collections.defaultdict(list)
+
+        # Combine report files into a single dict. If there are duplicate annotations, make sure we have the superset
+        # of data.
+        for r in self.report_files:
+            self._add_report_file_to_full_report(r, report)
+
+        return report
+
+    def _write_doc_file(self, doc_filename, doc_data):
+        """
+        Write out a single report file with the given data. This is rendered using the configured top level template.
+
+        Args:
+            doc_filename: Filename to write to.
+            doc_data: Dict of reporting data to use, in the {'file name': [list, of, annotations,]} style.
+        """
+        full_doc_filename = os.path.join(
+            self.config.rendered_report_dir,
+            slugify(doc_filename)
+        )
+
+        full_doc_filename += self.config.rendered_report_file_extension
+
+        self.echo.echo_v('Writing {}'.format(full_doc_filename))
+
+        with open(full_doc_filename, 'w') as output:
+            output.write(self.top_level_template.render(
+                create_time=self.create_time,
+                report=doc_data,
+                all_choices=self.all_choices,
+                all_annotations=self.config.annotation_tokens,
+                group_mapping=self.group_mapping,
+                slugify=slugify,
+                source_link_prefix=self.config.rendered_report_source_link_prefix)
+            )
+
+    def _generate_per_choice_docs(self):
+        """
+        Generate a page of documentation for each configured annotation choice.
+        """
+        for choice in self.all_choices:
+            choice_report = collections.defaultdict(list)
+            for filename in self.full_report:
+                for annotation in self.full_report[filename]:
+                    if isinstance(annotation['annotation_data'], list) and choice in annotation['annotation_data']:
+                        choice_report[filename].append(annotation)
+
+            self._write_doc_file('choice_{}'.format(choice), choice_report)
+
+    def _generate_per_annotation_docs(self):
+        """
+        Generate a page of documentation for each configured annotation.
+        """
+        for annotation in self.config.annotation_tokens:
+            annotation_report = collections.defaultdict(list)
+            for filename in self.full_report:
+                for report_annotation in self.full_report[filename]:
+                    if report_annotation['annotation_token'] == annotation:
+                        annotation_report[filename].append(report_annotation)
+
+            self._write_doc_file('annotation_{}'.format(annotation), annotation_report)
+
+    def render(self):
+        """
+        Perform the rendering of all documentation using the configured Jinja2 templates.
+        """
+        # Generate the top level list of all annotations
+        self._write_doc_file('index', self.full_report)
+        self._generate_per_choice_docs()
+        self._generate_per_annotation_docs()

code_annotations/helpers.py

@@ -3,10 +3,8 @@
 """
 import os
 import sys
-from collections import OrderedDict
 
 import click
-import yaml
 
 
 def fail(msg):
@@ -20,76 +18,6 @@ def fail(msg):
     sys.exit(-1)
 
 
-def yaml_ordered_load(stream):
-    """
-    Load YAML files in an ordered way.
-
-    We use this to maintain the order of annotations in the safelist. Slighty modified from
-    https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts/21048064#21048064
-
-    Args:
-        stream: File-like handle to load
-
-    Returns:
-        Ordered Python representation of the YAML file
-    """
-    class OrderedLoader(yaml.SafeLoader):
-        """
-        A dummy object that we can safely modify using `add_constructor`.
-        """
-
-        pass
-
-    def construct_mapping(loader, node):
-        """
-        Handle actually ordering the data on a node-by-node basis.
-
-        Args:
-            loader: A PyYAML resolver
-            node: The node to be constructed
-
-        Returns:
-            OrderedDict of the mapped pairs
-        """
-        loader.flatten_mapping(node)
-        return OrderedDict(loader.construct_pairs(node))
-
-    OrderedLoader.add_constructor(
-        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-        construct_mapping
-    )
-
-    return yaml.load(stream, OrderedLoader)
-
-
-def yaml_ordered_dump(data, stream, **kwargs):
-    """
-    Dump data to YAML files in an ordered way.
-
-    We use this to maintain the order of annotations in the safelist. Slighty modified from
-    https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts/21048064#21048064
-
-    Args:
-        data: Python object to be dumped
-        stream: File-like handle to write to
-        **kwargs:
-
-    Returns:
-        Results of the yaml.dump
-    """
-    class OrderedDumper(yaml.SafeDumper):
-        pass
-
-    def _dict_representer(dumper, data):
-        return dumper.represent_mapping(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-            data.items()
-        )
-
-    OrderedDumper.add_representer(OrderedDict, _dict_representer)
-    return yaml.dump(data, stream, OrderedDumper, **kwargs)
-
-
 class VerboseEcho(object):
     """
     Helper to handle verbosity-dependent logging.
@@ -177,7 +105,7 @@ def clean_abs_path(filename_to_clean, parent_path):
     # If we are operating on only one file we don't know what to strip off here,
     # just return the whole thing.
     if filename_to_clean == parent_path:
-        return parent_path
+        return os.path.basename(filename_to_clean)
     return os.path.relpath(filename_to_clean, parent_path)
 
 

code_annotations/report_templates/annotation.tpl

@@ -0,0 +1,5 @@
+{% if annotation.extra and annotation.extra.object_id %}
+`<{{ annotation.extra.object_id }}> line {{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
+{% else %}
+`{{ filename }}:{{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
+{% endif %}