Use sphinx-apidoc to create API reference (GH-309)

* Add some missing files to .gitignore
* Remove duplicate open_in_browser from lxml.html.__all__
* Make ETreeXMLSchemaTestCase docstring Sphinx autodoc friendly
* Fix outdated codespeak.net links in docstrings
* Convert html/defs.py comment to be the  module docstring
* Use sphinx-apidoc to create the API reference instead of epydoc
  Epydoc is Python 2 only and unmaintained.
  sphinx-apidoc is run before the build step, to avoid duplicate entries being created.
* Include the elements from html.builder in the API reference
* Use Python 3.8 for coverage Travis job
* Build html documentation in Travis

.gitignore

@@ -16,9 +16,14 @@ libs
 *.pyd
 MANIFEST
 
+doc/api/lxml*.rst
+doc/api/_build/
+doc/s5/lxml-ep2008.html
 src/lxml/includes/lxml-version.h
 src/lxml/*.html
 src/lxml/html/*.c
+src/lxml/_elementpath.c
+src/lxml/builder.c
 src/lxml/etree.c
 src/lxml/etree.h
 src/lxml/etree_api.h
@@ -27,3 +32,4 @@ src/lxml/lxml.etree.h
 src/lxml/lxml.etree_api.h
 src/lxml/objectify.c
 src/lxml/lxml.objectify.c
+src/lxml/sax.c

.travis.yml

@@ -29,8 +29,15 @@ env:
 
 matrix:
   include:
-    - python: 3.7
-      env: STATIC_DEPS=false EXTRA_DEPS="coverage<5"
+    - python: 3.8
+      env:
+        - STATIC_DEPS=false
+        - EXTRA_DEPS="docutils pygments sphinx sphinx-rtd-theme"
+      script: make html
+    - python: 3.8
+      env:
+        - STATIC_DEPS=false
+        - EXTRA_DEPS="coverage<5"
     - python: 3.8
       env:
         - STATIC_DEPS=true

Makefile

@@ -105,34 +105,33 @@ ftest_build: build
 ftest_inplace: inplace
 	$(PYTHON) test.py -f $(TESTFLAGS) $(TESTOPTS)
 
-apihtml: inplace
-	rm -fr doc/html/api
-	@[ -x "`which epydoc`" ] \
-		&& (cd src && echo "Generating API docs ..." && \
-			PYTHONPATH=. epydoc -v --docformat "restructuredtext en" \
-			-o ../doc/html/api --exclude='[.]html[.]tests|[.]_' \
-			--exclude-introspect='[.]usedoctest' \
-			--name "lxml API" --url / lxml/) \
-		|| (echo "not generating epydoc API documentation")
+apidoc: clean docclean inplace3
+	@[ -x "`which sphinx-apidoc`" ] \
+		&& (echo "Generating API docs ..." && \
+			PYTHONPATH=src:$(PYTHONPATH) sphinx-apidoc -e -P -T -o doc/api src/lxml \
+				"*includes" "*tests" "*pyclasslookup.py" "*usedoctest.py" "*html/_html5builder.py") \
+		|| (echo "not generating Sphinx autodoc API rst files")
+
+apihtml: apidoc
+	@[ -x "`which sphinx-build`" ] \
+		&& (echo "Generating API docs ..." && \
+			make -C doc/api html) \
+		|| (echo "not generating Sphinx autodoc API documentation")
 
-website: inplace
-	PYTHONPATH=src:$(PYTHONPATH) $(PYTHON) doc/mkhtml.py doc/html . ${LXMLVERSION}
+website: inplace3
+	PYTHONPATH=src:$(PYTHONPATH) $(PYTHON3) doc/mkhtml.py doc/html . ${LXMLVERSION}
 
-html: inplace website apihtml s5
+html: apihtml website s5
 
 s5:
 	$(MAKE) -C doc/s5 slides
 
-apipdf: inplace
-	rm -fr doc/pdf
-	mkdir -p doc/pdf
-	@[ -x "`which epydoc`" ] \
-		&& (cd src && echo "Generating API docs ..." && \
-			PYTHONPATH=. epydoc -v --latex --docformat "restructuredtext en" \
-			-o ../doc/pdf --exclude='([.]html)?[.]tests|[.]_' \
-			--exclude-introspect='html[.]clean|[.]usedoctest' \
-			--name "lxml API" --url / lxml/) \
-		|| (echo "not generating epydoc API documentation")
+apipdf: apidoc
+	rm -fr doc/api/_build
+	@[ -x "`which sphinx-build`" ] \
+		&& (echo "Generating API PDF docs ..." && \
+			make -C doc/api latexpdf) \
+		|| (echo "not generating Sphinx autodoc API PDF documentation")
 
 pdf: apipdf
 	$(PYTHON) doc/mklatex.py doc/pdf . ${LXMLVERSION}
@@ -164,6 +163,8 @@ docclean:
 	$(MAKE) -C doc/s5 clean
 	rm -f doc/html/*.html
 	rm -fr doc/html/api
+	rm -f doc/api/lxml*.rst
+	rm -fr doc/api/_build
 	rm -fr doc/pdf
 
 realclean: clean docclean

doc/api/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+html:
+	@$(SPHINXBUILD) -b html "$(SOURCEDIR)" -d "$(BUILDDIR)/doctrees" ../html/api $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/api/conf.py

@@ -0,0 +1,56 @@
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../src'))
+
+from lxml import __version__ as lxml_version
+
+# -- Project information -----------------------------------------------------
+
+project = 'lxml'
+copyright = '2020, lxml dev team'
+author = 'lxml dev team'
+version = lxml_version
+
+
+# -- General configuration ---------------------------------------------------
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+    'sphinx_rtd_theme',
+]
+
+language = 'en'
+
+exclude_patterns = ['_build']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+html_theme = 'sphinx_rtd_theme'
+
+html_logo = '../html/python-xml.png'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+html_theme_options = {
+    'collapse_navigation': False,
+    'titles_only': True,
+}
+
+# -- Extension configuration -------------------------------------------------
+
+autodoc_default_options = {
+    'ignore-module-all': True,
+    'private-members': True,
+}
+
+autodoc_member_order = 'groupwise'
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+#todo_include_todos = True

doc/api/index.rst

@@ -0,0 +1,14 @@
+lxml API Reference
+==================
+
+.. toctree::
+   :maxdepth: 4
+
+   lxml
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

src/lxml/classlookup.pxi

@@ -504,7 +504,7 @@ cdef class PythonElementClassLookup(FallbackElementClassLookup):
     `lxml.etree` API (such as XPath, extended slicing or some
     iteration methods).
 
-    See http://codespeak.net/lxml/element_classes.html
+    See https://lxml.de/element_classes.html
     """
     def __cinit__(self):
         self._lookup_function = _python_class_lookup

src/lxml/html/__init__.py

@@ -37,7 +37,7 @@
     'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring',
     'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form',
     'find_rel_links', 'find_class', 'make_links_absolute',
-    'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser', 'parse']
+    'resolve_base_href', 'iterlinks', 'rewrite_links', 'parse']
 
 
 import copy