From 13e5bfb73ccc8e45b103bc2a459d67aaa35991f2 Mon Sep 17 00:00:00 2001 From: Cory Gwin Date: Fri, 12 Nov 2021 19:35:27 +0000 Subject: [PATCH] Improve the speed of valdiation Motivation: - We are looking to build out a notebook viewer with an eye toward performance. In our flamegraph research we found that as much as 90% of the time spent in Notebook creation can occur in the preprocessor valdiation logic. This pr introduces the optional idea of optimistic validation, in which we care not about which preprocessor introduces a validation error but that the notebook ends in a valid state, since we care more about speed. Related Issues: - https://github.com/jupyter/nbconvert/issues/1663 Changes: - Introduces a config option to remove some of the validation checks. Questions: - I am new to the project and am not sure where a test should live for this logic, or the best way to structure such a test. If a reference is available I will happily added a test. --- nbconvert/exporters/exporter.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/nbconvert/exporters/exporter.py b/nbconvert/exporters/exporter.py index 1c487ab56..bb6dffa91 100644 --- a/nbconvert/exporters/exporter.py +++ b/nbconvert/exporters/exporter.py @@ -61,6 +61,10 @@ class Exporter(LoggingConfigurable): help="Extension of the file that should be written to disk" ).tag(config=True) + optimistic_validation = Bool(False, + help = "Reduces the number of validation steps so that it only occurs after all preprocesors have run." + ).tag(config=True) + # MIME type of the result file, for HTTP response headers. # This is *not* a traitlet, because we want to be able to access it from # the class, not just on instances. @@ -296,6 +300,13 @@ def _init_resources(self, resources): resources['output_extension'] = self.file_extension return resources + def _validate_preprocessor(self, nbc, preprocessor): + try: + nbformat.validate(nbc, relax_add_props=True) + except nbformat.ValidationError: + self.log.error('Notebook is invalid after preprocessor %s', + preprocessor) + raise def _preprocess(self, nb, resources): """ @@ -321,11 +332,10 @@ def _preprocess(self, nb, resources): # to each preprocessor for preprocessor in self._preprocessors: nbc, resc = preprocessor(nbc, resc) - try: - nbformat.validate(nbc, relax_add_props=True) - except nbformat.ValidationError: - self.log.error('Notebook is invalid after preprocessor %s', - preprocessor) - raise + if not self.optimistic_validation: + self._validate_preprocessor(nbc, preprocessor) + + if self.optimistic_validation: + self._validate_preprocessor(nbc, preprocessor) return nbc, resc