Skip to content

Commit

Permalink
Revert experimentation
Browse files Browse the repository at this point in the history
  • Loading branch information
AA-Turner committed May 23, 2022
1 parent b983eb5 commit 0f5c22c
Show file tree
Hide file tree
Showing 3 changed files with 274 additions and 234 deletions.
7 changes: 3 additions & 4 deletions sphinx/search/en.py
Expand Up @@ -2,9 +2,8 @@

from typing import Dict

import snowballstemmer

from sphinx.search import SearchLanguage
from sphinx.util.stemmer import get_stemmer

english_stopwords = set("""
a and are as at
Expand Down Expand Up @@ -212,7 +211,7 @@ class SearchEnglish(SearchLanguage):
stopwords = english_stopwords

def init(self, options: Dict) -> None:
self.stemmer = snowballstemmer.stemmer('english')
self.stemmer = get_stemmer()

def stem(self, word: str) -> str:
return self.stemmer.stemWord(word.lower())
return self.stemmer.stem(word.lower())
31 changes: 26 additions & 5 deletions sphinx/util/stemmer/__init__.py
@@ -1,16 +1,37 @@
"""Word stemming utilities for Sphinx."""

import snowballstemmer

from sphinx.util.stemmer.porter import PorterStemmer

try:
from Stemmer import Stemmer as _PyStemmer
PYSTEMMER = True
except ImportError:
PYSTEMMER = False


class BaseStemmer:
def stem(self, word: str) -> str:
raise NotImplementedError
raise NotImplementedError()


def get_stemmer() -> BaseStemmer:
stemmer = snowballstemmer.stemmer('english')
class PyStemmer(BaseStemmer):
def __init__(self) -> None:
self.stemmer = _PyStemmer('porter')

def stem(self, word: str) -> str:
return self.stemmer.stemWord(word)


class StandardStemmer(PorterStemmer, BaseStemmer):
"""All those porter stemmer implementations look hideous;
make at least the stem method nicer.
"""
def stem(self, word: str) -> str: # type: ignore
return super().stem(word, 0, len(word) - 1)


def get_stemmer() -> BaseStemmer:
if PYSTEMMER:
return PyStemmer()
else:
return StandardStemmer()

0 comments on commit 0f5c22c

Please sign in to comment.