diff --git a/.gitignore b/.gitignore index 60645072997..35fd23178f5 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ doc/_build/ doc/locale/ tests/.coverage tests/build/ +tests/js/roots/*/_build tests/test-server.lock utils/regression_test.js diff --git a/.ruff.toml b/.ruff.toml index 4c0bb2d211a..a64a00012fe 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -4,6 +4,7 @@ output-format = "full" extend-exclude = [ "tests/roots/*", + "tests/js/roots/*", "build/*", "doc/_build/*", "sphinx/search/*", diff --git a/CHANGES.rst b/CHANGES.rst index 3e227b53371..3d021fb8eb2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -49,6 +49,9 @@ Bugs fixed Testing ------- +* karma: refactor HTML search tests to use fixtures generated by Sphinx. + Patch by James Addison. + Release 7.3.7 (released Apr 19, 2024) ===================================== diff --git a/doc/internals/contributing.rst b/doc/internals/contributing.rst index b0c5b9568f5..eac35606a3f 100644 --- a/doc/internals/contributing.rst +++ b/doc/internals/contributing.rst @@ -338,3 +338,9 @@ Debugging tips Minified files in ``sphinx/search/minified-js/*.js`` are generated from non-minified ones using ``uglifyjs`` (installed via npm), with ``-m`` option to enable mangling. + +* The ``searchindex.js`` files found in the ``tests/js/fixtures/*`` directories + are generated by using the standard Sphinx HTML builder on the corresponding + input projects found in ``tests/js/roots/*``. The fixtures provide test data + used by the Sphinx JavaScript unit tests, and can be regenerated by running + the ``utils/generate_js_fixtures.py`` script. diff --git a/karma.conf.js b/karma.conf.js index 8a18e80ba7a..4f1b9c616e4 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,7 +15,9 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ + { pattern: 'tests/js/fixtures/**/*.js', included: false, served: true }, 'tests/js/documentation_options.js', + 'tests/js/language_data.js', 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js new file mode 100644 index 00000000000..f704f7aa7e2 --- /dev/null +++ b/tests/js/fixtures/cpp/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "becaus": 0, "c": 0, "can": 0, "cardin": 0, "challeng": 0, "charact": 0, "class": 0, "descript": 0, "drop": 0, "engin": 0, "fixtur": 0, "frequent": 0, "gener": 0, "i": 0, "index": 0, "inflat": 0, "mathemat": 0, "occur": 0, "often": 0, "project": 0, "punctuat": 0, "queri": 0, "relat": 0, "sampl": 0, "search": 0, "size": 0, "sphinx": 0, "term": 0, "thei": 0, "thi": 0, "token": 0, "us": 0, "web": 0, "would": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/fixtures/multiterm/searchindex.js b/tests/js/fixtures/multiterm/searchindex.js new file mode 100644 index 00000000000..b791df93d11 --- /dev/null +++ b/tests/js/fixtures/multiterm/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"At": 0, "adjac": 0, "all": 0, "an": 0, "appear": 0, "applic": 0, "ar": 0, "built": 0, "can": 0, "check": 0, "contain": 0, "do": 0, "document": 0, "doesn": 0, "each": 0, "fixtur": 0, "format": 0, "function": 0, "futur": 0, "html": 0, "i": 0, "includ": 0, "match": 0, "messag": 0, "multipl": 0, "multiterm": 0, "order": 0, "other": 0, "output": 0, "perform": 0, "perhap": 0, "phrase": 0, "project": 0, "queri": 0, "requir": 0, "same": 0, "search": 0, "successfulli": 0, "support": 0, "t": 0, "term": 0, "test": 0, "thi": 0, "time": 0, "us": 0, "when": 0, "write": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file diff --git a/tests/js/fixtures/partial/searchindex.js b/tests/js/fixtures/partial/searchindex.js new file mode 100644 index 00000000000..6ccfbd6d07e --- /dev/null +++ b/tests/js/fixtures/partial/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {"sphinx_utils module": [[0, "sphinx-utils-module"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"also": 0, "ar": 0, "built": 0, "confirm": 0, "document": 0, "function": 0, "html": 0, "i": 0, "includ": 0, "input": 0, "javascript": 0, "known": 0, "match": 0, "partial": 0, "possibl": 0, "prefix": 0, "project": 0, "provid": 0, "restructuredtext": 0, "sampl": 0, "search": 0, "should": 0, "thi": 0, "titl": 0, "us": 0, "when": 0}, "titles": ["sphinx_utils module"], "titleterms": {"modul": 0, "sphinx_util": 0}}) \ No newline at end of file diff --git a/tests/js/language_data.js b/tests/js/language_data.js new file mode 100644 index 00000000000..89083d9ec7c --- /dev/null +++ b/tests/js/language_data.js @@ -0,0 +1,26 @@ +/* + * language_data.js + * ~~~~~~~~~~~~~~~~ + * + * This script contains the language-specific data used by searchtools.js, + * namely the list of stopwords, stemmer, scorer and splitter. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +var stopwords = []; + + +/* Non-minified version is copied as a separate JS file, if available */ + +/** + * Dummy stemmer for languages without stemming rules. + */ +var Stemmer = function() { + this.stemWord = function(w) { + return w; + } +} + diff --git a/tests/js/roots/cpp/conf.py b/tests/js/roots/cpp/conf.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/js/roots/cpp/index.rst b/tests/js/roots/cpp/index.rst new file mode 100644 index 00000000000..d731343dca6 --- /dev/null +++ b/tests/js/roots/cpp/index.rst @@ -0,0 +1,10 @@ +This is a sample C++ project used to generate a search engine index fixture. + +.. cpp:class:: public Sphinx + + The description of Sphinx class. + +Indexing and querying the term C++ can be challenging, because search-related +tokenization often drops punctuation and mathematical characters (they occur +frequently on the web and would inflate the cardinality and size of web search +indexes). diff --git a/tests/js/roots/multiterm/conf.py b/tests/js/roots/multiterm/conf.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/js/roots/multiterm/index.rst b/tests/js/roots/multiterm/index.rst new file mode 100644 index 00000000000..495e5ce858c --- /dev/null +++ b/tests/js/roots/multiterm/index.rst @@ -0,0 +1,13 @@ +Main Page +========= + +This is the main page of the ``multiterm`` test project. + +This document is used as a test fixture to check that the search functionality +included when projects are built into an HTML output format can successfully +match this document when a search query containing multiple terms is performed. + +At the time-of-writing this message, the application doesn't support "phrase +queries" -- queries that require all of the contained terms to appear adjacent +to each other and in the same order in the document as in the query; perhaps it +will do in future? diff --git a/tests/js/roots/partial/conf.py b/tests/js/roots/partial/conf.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/js/roots/partial/index.rst b/tests/js/roots/partial/index.rst new file mode 100644 index 00000000000..6a9561b3994 --- /dev/null +++ b/tests/js/roots/partial/index.rst @@ -0,0 +1,9 @@ +sphinx_utils module +=================== + +Partial (also known as "prefix") matches on document titles should be possible +using the JavaScript search functionality included when HTML documentation +projects are built. + +This document provides a sample reStructuredText input to confirm that partial +title matching is possible. diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 99ebdafb1de..5e97572fb3e 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -1,20 +1,20 @@ describe('Basic html theme search', function() { + function loadFixture(name) { + req = new XMLHttpRequest(); + req.open("GET", `base/tests/js/fixtures/${name}`, false); + req.send(null); + return req.responseText; + } + describe('terms search', function() { it('should find "C++" when in index', function() { - index = { - docnames:["index"], - filenames:["index.rst"], - terms:{'c++':0}, - titles:["<no title>"], - titleterms:{} - } - Search.setIndex(index); - searchterms = ['c++']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + eval(loadFixture("cpp/searchindex.js")); + + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('C++'); + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ "index", @@ -28,22 +28,11 @@ describe('Basic html theme search', function() { }); it('should be able to search for multiple terms', function() { - index = { - alltitles: { - 'Main Page': [[0, 'main-page']], - }, - docnames:["index"], - filenames:["index.rst"], - terms:{main:0, page:0}, - titles:["Main Page"], - titleterms:{ main:0, page:0 } - } - Search.setIndex(index); - - searchterms = ['main', 'page']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + eval(loadFixture("multiterm/searchindex.js")); + + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('main page'); + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ 'index', 'Main Page', @@ -55,18 +44,11 @@ describe('Basic html theme search', function() { }); it('should partially-match "sphinx" when in title index', function() { - index = { - docnames:["index"], - filenames:["index.rst"], - terms:{'useful': 0, 'utilities': 0}, - titles:["sphinx_utils module"], - titleterms:{'sphinx_utils': 0} - } - Search.setIndex(index); - searchterms = ['sphinx']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + eval(loadFixture("partial/searchindex.js")); + + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('sphinx'); + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ "index", @@ -81,6 +63,37 @@ describe('Basic html theme search', function() { }); + describe('aggregation of search results', function() { + + it('should combine document title and document term matches', function() { + eval(loadFixture("multiterm/searchindex.js")); + + searchParameters = Search._parseQuery('main page'); + + // fixme: duplicate result due to https://github.com/sphinx-doc/sphinx/issues/11961 + hits = [ + [ + 'index', + 'Main Page', + '', + null, + 15, + 'index.rst' + ], + [ + 'index', + 'Main Page', + '#main-page', + null, + 100, + 'index.rst' + ] + ]; + expect(Search._performSearch(...searchParameters)).toEqual(hits); + }); + + }); + }); describe("htmlToText", function() { diff --git a/tests/test_search.py b/tests/test_search.py index 63443a8b053..d36bb57743c 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -11,6 +11,8 @@ from sphinx.search import IndexBuilder +from tests.utils import TESTS_ROOT + class DummyEnvironment: def __init__(self, version, domains): @@ -346,3 +348,17 @@ def assert_is_sorted(item, path: str): assert item == sorted(item), f'{err_path} is not sorted' for i, child in enumerate(item): assert_is_sorted(child, f'{path}[{i}]') + + +@pytest.mark.parametrize('directory', ( + directory for directory in (TESTS_ROOT / 'js' / 'roots').iterdir() +)) +def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): + app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name) + app.build() + + fresh_searchindex = (app.outdir / 'searchindex.js') + existing_searchindex = (TESTS_ROOT / 'js' / 'fixtures' / directory.name / 'searchindex.js') + + msg = f"Search index fixture {existing_searchindex} does not match regenerated copy." + assert fresh_searchindex.read_bytes() == existing_searchindex.read_bytes(), msg diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py new file mode 100755 index 00000000000..37e844f1a80 --- /dev/null +++ b/utils/generate_js_fixtures.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import subprocess +from pathlib import Path + +SPHINX_ROOT = Path(__file__).resolve().parent.parent +TEST_JS_FIXTURES = SPHINX_ROOT / 'tests' / 'js' / 'fixtures' +TEST_JS_ROOTS = SPHINX_ROOT / 'tests' / 'js' / 'roots' + + +def build(srcdir: Path) -> None: + cmd = ( + 'sphinx-build', + '--fresh-env', + '--quiet', + *('--builder', 'html'), + f'{srcdir}', + f'{srcdir}/_build', + ) + subprocess.run(cmd, check=True, capture_output=True) + + +for directory in TEST_JS_ROOTS.iterdir(): + searchindex = directory / '_build' / 'searchindex.js' + destination = TEST_JS_FIXTURES / directory.name / 'searchindex.js' + + print(f'Building {directory} ... ', end='') + build(directory) + print('done') + + print(f'Moving {searchindex} to {destination} ... ', end='') + destination.parent.mkdir(exist_ok=True) + searchindex.replace(destination) + print('done')