Leo Galambos
5e3d3c7c7d
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
361 lines
11 KiB
Python
361 lines
11 KiB
Python
###############################################################################
|
|
# Copyright (C) 2026, Leo Galambos
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are met:
|
|
#
|
|
# 1. Redistributions of source code must retain the above copyright notice,
|
|
# this list of conditions and the following disclaimer.
|
|
#
|
|
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
#
|
|
# 3. Neither the name of the copyright holder nor the names of its contributors
|
|
# may be used to endorse or promote products derived from this software
|
|
# without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
# POSSIBILITY OF SUCH DAMAGE.
|
|
###############################################################################
|
|
|
|
"""Stemmer engine adapters for the benchmark.
|
|
|
|
Every engine exposes a uniform interface:
|
|
|
|
engine.name -> str
|
|
engine.kind -> "native-batch" | "c-batch" | "py-batch" | "py-loop"
|
|
engine.supports(code) -> bool (ISO-639-1 language code)
|
|
engine.make(code) -> callable(list[str]) -> list[str] (the batch fn)
|
|
|
|
``kind`` records how batching is implemented:
|
|
|
|
- ``native-batch``: one native batch call
|
|
- ``c-batch``: C extension with a list entry point
|
|
- ``py-batch``: pure-Python object with a list method
|
|
- ``py-loop``: repeated scalar calls in Python
|
|
|
|
Engines whose package is not installed are simply reported as unavailable, so
|
|
the benchmark runs with whatever the user has.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
from pathlib import Path
|
|
from typing import Callable, Optional
|
|
|
|
|
|
def _module_info(obj) -> dict:
|
|
"""Provenance of the module that actually provides ``obj``.
|
|
|
|
The returned metadata identifies the backing module and distinguishes
|
|
native extensions from pure-Python implementations.
|
|
|
|
Resolves via ``type(obj).__module__`` -> ``sys.modules`` because
|
|
``inspect.getmodule`` returns ``None`` for Cython extension instances.
|
|
"""
|
|
import sys
|
|
|
|
name = type(obj).__module__
|
|
mod = sys.modules.get(name) or inspect.getmodule(obj)
|
|
file = getattr(mod, "__file__", None) or ""
|
|
lower = file.lower()
|
|
compiled = (
|
|
lower.endswith((".pyd", ".so", ".dll")) or "cpython" in lower or "abi3" in lower
|
|
)
|
|
return {
|
|
"backing_module": name,
|
|
"backing_file": file,
|
|
"compiled_extension": compiled,
|
|
}
|
|
|
|
|
|
# ISO-639-1 -> Snowball algorithm name (matches the Java SnowballLanguageCase
|
|
# mapping; nb/nn both map to the single Snowball "norwegian" algorithm).
|
|
_SNOWBALL_NAMES: dict[str, str] = {
|
|
"cs": "czech",
|
|
"fa": "persian",
|
|
"pl": "polish",
|
|
"da": "danish",
|
|
"nl": "dutch",
|
|
"en": "english",
|
|
"fi": "finnish",
|
|
"fr": "french",
|
|
"de": "german",
|
|
"hu": "hungarian",
|
|
"it": "italian",
|
|
"nb": "norwegian",
|
|
"nn": "norwegian",
|
|
"pt": "portuguese",
|
|
"ru": "russian",
|
|
"es": "spanish",
|
|
"sv": "swedish",
|
|
"yi": "yiddish",
|
|
}
|
|
|
|
BatchFn = Callable[[list[str]], list[str]]
|
|
|
|
|
|
class Engine:
|
|
name: str = "engine"
|
|
kind: str = "py-loop"
|
|
|
|
def available(self) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def supports(self, code: str) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
raise NotImplementedError
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
return {
|
|
"backing_module": self.name,
|
|
"backing_file": "",
|
|
"compiled_extension": False,
|
|
"algorithm": None,
|
|
}
|
|
|
|
|
|
class RadixorEngine(Engine):
|
|
name = "radixor"
|
|
kind = "native-batch"
|
|
|
|
def __init__(self, lowercase: bool = False) -> None:
|
|
# The shared corpus is already lowercase. Disable Radixor's redundant
|
|
# normalization unless a benchmark explicitly includes that cost.
|
|
self._lowercase = lowercase
|
|
|
|
def available(self) -> bool:
|
|
try:
|
|
import radixor # noqa: F401
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def supports(self, code: str) -> bool:
|
|
try:
|
|
from radixor import _LANGUAGE_ALIASES
|
|
|
|
return code in _LANGUAGE_ALIASES
|
|
except Exception:
|
|
return False
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
from radixor import Stemmer
|
|
|
|
s = Stemmer(code, lowercase=self._lowercase, cache_size=0)
|
|
return s.stem_batch
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
from radixor import Stemmer
|
|
|
|
s = Stemmer(code, lowercase=self._lowercase, cache_size=0)
|
|
info = _module_info(s._core)
|
|
info["algorithm"] = "radixor-trie"
|
|
info["lowercase"] = self._lowercase
|
|
info["cache_disabled"] = True
|
|
return info
|
|
|
|
|
|
class PyStemmerEngine(Engine):
|
|
name = "PyStemmer"
|
|
kind = "c-batch"
|
|
|
|
def available(self) -> bool:
|
|
try:
|
|
import Stemmer # noqa: F401
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def _algorithms(self) -> set[str]:
|
|
import Stemmer
|
|
|
|
return {a.lower() for a in Stemmer.algorithms()}
|
|
|
|
def supports(self, code: str) -> bool:
|
|
name = _SNOWBALL_NAMES.get(code)
|
|
return bool(name) and name in self._algorithms()
|
|
|
|
@staticmethod
|
|
def _new_stemmer(code: str):
|
|
import Stemmer
|
|
|
|
stemmer = Stemmer.Stemmer(_SNOWBALL_NAMES[code])
|
|
# Repeated passes would otherwise measure PyStemmer's default cache
|
|
# after the first pass. Both native engines therefore run uncached.
|
|
stemmer.maxCacheSize = 0
|
|
return stemmer
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
stemmer = self._new_stemmer(code)
|
|
# PyStemmer's native list entry point: one C call for the whole batch.
|
|
return stemmer.stemWords
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
stemmer = self._new_stemmer(code)
|
|
info = _module_info(stemmer)
|
|
info["algorithm"] = _SNOWBALL_NAMES[code]
|
|
info["cache_disabled"] = True
|
|
# Record whether PyStemmer resolves to an independent native extension.
|
|
try:
|
|
import snowballstemmer
|
|
|
|
snowball_dir = str(Path(snowballstemmer.__file__).resolve().parent).lower()
|
|
except Exception:
|
|
snowball_dir = None
|
|
backing = info["backing_file"].lower()
|
|
info["independent_of_snowballstemmer"] = bool(
|
|
info["compiled_extension"]
|
|
and (snowball_dir is None or snowball_dir not in backing)
|
|
)
|
|
return info
|
|
|
|
|
|
class SnowballStemmerEngine(Engine):
|
|
"""Pure-Python Snowball backend.
|
|
|
|
``snowballstemmer.stemmer()`` delegates to PyStemmer (the C extension) when
|
|
PyStemmer is installed, which would make this engine a duplicate of
|
|
``PyStemmer``. To retain a distinct implementation, this adapter imports
|
|
the language's pure-Python class directly from
|
|
``snowballstemmer.<name>_stemmer``.
|
|
"""
|
|
|
|
name = "snowballstemmer-pure"
|
|
kind = "py-batch"
|
|
|
|
def available(self) -> bool:
|
|
try:
|
|
import snowballstemmer # noqa: F401
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def _load_class(self, code: str):
|
|
import importlib
|
|
|
|
name = _SNOWBALL_NAMES.get(code)
|
|
if not name:
|
|
return None
|
|
module = importlib.import_module(f"snowballstemmer.{name}_stemmer")
|
|
class_name = name.capitalize() + "Stemmer"
|
|
return getattr(module, class_name, None)
|
|
|
|
def supports(self, code: str) -> bool:
|
|
try:
|
|
return self._load_class(code) is not None
|
|
except Exception:
|
|
return False
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
stemmer = self._load_class(code)()
|
|
return stemmer.stemWords # pure-Python loop over the list, internally
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
stemmer = self._load_class(code)()
|
|
info = _module_info(stemmer)
|
|
info["algorithm"] = _SNOWBALL_NAMES[code]
|
|
return info
|
|
|
|
|
|
class NltkPorterEngine(Engine):
|
|
name = "nltk-porter"
|
|
kind = "py-loop"
|
|
|
|
def available(self) -> bool:
|
|
try:
|
|
from nltk.stem import PorterStemmer # noqa: F401
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def supports(self, code: str) -> bool:
|
|
return code == "en" # Porter is English-only
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
from nltk.stem import PorterStemmer
|
|
|
|
ps = PorterStemmer()
|
|
stem = ps.stem
|
|
|
|
def batch(words: list[str]) -> list[str]:
|
|
return [stem(w) for w in words]
|
|
|
|
return batch
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
from nltk.stem import PorterStemmer
|
|
|
|
info = _module_info(PorterStemmer())
|
|
info["algorithm"] = "porter"
|
|
return info
|
|
|
|
|
|
class CistemEngine(Engine):
|
|
"""CISTEM — a fast lightweight German stemmer (German only)."""
|
|
|
|
name = "cistem"
|
|
kind = "py-loop"
|
|
|
|
def available(self) -> bool:
|
|
try:
|
|
from nltk.stem.cistem import Cistem # noqa: F401
|
|
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def supports(self, code: str) -> bool:
|
|
return code == "de"
|
|
|
|
def make(self, code: str) -> BatchFn:
|
|
from nltk.stem.cistem import Cistem
|
|
|
|
stem = Cistem().stem
|
|
|
|
def batch(words: list[str]) -> list[str]:
|
|
return [stem(w) for w in words]
|
|
|
|
return batch
|
|
|
|
def provenance(self, code: str) -> dict:
|
|
from nltk.stem.cistem import Cistem
|
|
|
|
info = _module_info(Cistem())
|
|
info["algorithm"] = "cistem"
|
|
return info
|
|
|
|
|
|
ALL_ENGINES: list[Engine] = [
|
|
RadixorEngine(),
|
|
PyStemmerEngine(),
|
|
SnowballStemmerEngine(),
|
|
NltkPorterEngine(),
|
|
CistemEngine(),
|
|
]
|
|
|
|
|
|
def available_engines(names: Optional[set[str]] = None) -> list[Engine]:
|
|
engines = [e for e in ALL_ENGINES if e.available()]
|
|
if names:
|
|
engines = [e for e in engines if e.name in names]
|
|
return engines
|