Modernize nltk.org/howto pages

.gitignore

@@ -40,6 +40,7 @@ coverage.xml
 
 # automatically built files for website
 web/api/*.rst
+web/howto/*.rst
 
 # iPython notebooks
 

nltk/test/corpus.doctest

@@ -25,7 +25,7 @@ and new corpus reader classes.  Section `Regression Tests`_
 and associated functions and classes.
 
 .. contents:: **Table of Contents**
-  :depth: 2
+  :depth: 4
   :backlinks: none
 
 ---------------------

web/_templates/doctest.rst

@@ -0,0 +1,5 @@
+{% for item in range(17 + module_name|length) -%}#{%- endfor %}
+Sample usage for {{ module_name }}
+{% for item in range(17 + module_name|length) -%}#{%- endfor %}
+
+.. include:: ../../nltk/test/{{ module_name }}.doctest

web/conf.py

@@ -54,6 +54,46 @@ def run_apidoc(app):
     )
 
 
+def generate_howto():
+    """Custom function for generating contents in the ``howto`` folder,
+    based on the ``ntlk/test/*.doctest`` files.
+    """
+    import glob
+    import re
+
+    from jinja2 import Template
+
+    modules = []
+
+    web_folder = os.path.dirname(os.path.abspath(__file__))
+    howto_folder = os.path.join(web_folder, "howto")
+    if not os.path.exists(howto_folder):
+        os.makedirs(howto_folder)
+
+    # Load jinja template
+    with open(os.path.join(web_folder, "_templates", "doctest.rst")) as f:
+        doctest_template = Template(f.read())
+
+    print("Generating HOWTO pages...")
+    # Iterate over .doctest files, and find the module_name.
+    pattern = re.compile(r"(\w+)\.doctest$")
+    for path in glob.glob(os.path.join(web_folder, "..", "nltk", "test", "*.doctest")):
+        match = pattern.search(path)
+        module_name = match.group(1)
+        # Ignore index.doctest, we already have an index, i.e. howto.rst
+        if module_name == "index":
+            continue
+        # Write .rst files based on the doctest_template.
+        doctest_template.stream(module_name=module_name).dump(
+            os.path.join(howto_folder, f"{module_name}.rst")
+        )
+        modules.append(module_name)
+
+    print(f"Generated {len(modules)} HOWTO pages.")
+
+
+generate_howto()
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

web/howto.rst

@@ -0,0 +1,8 @@
+Example usage of NLTK modules
+=============================
+
+.. toctree::
+   :titlesonly:
+   :glob:
+
+   howto/*

web/index.rst

@@ -76,7 +76,7 @@ Next Steps
    :caption: NLTK Documentation
 
    API Reference <api/nltk>
-   Example Usage <https://www.nltk.org/howto>
+   Example Usage <howto>
    Module Index <py-modindex>
    Wiki <https://github.com/nltk/nltk/wiki>
    FAQ <https://github.com/nltk/nltk/wiki/FAQ>