Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(python): add native distribution and release infrastructure

Browse Source 
- add the Rust-backed Python API with PyStemmer compatibility
- distribute standard compiled models as a separate Python package
- generate model artifacts during builds instead of storing them in Git
- add GitHub release and Pages-backed package index workflows
- add Python tests, benchmarks, documentation, and Gradle integration
- refresh the documentation site, branding, and language benchmarks
This commit is contained in:
Leo Galambos
2026-08-10 22:34:32 +02:00
parent b45e143c84
commit 5e3d3c7c7d
139 changed files with 11420 additions and 747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
python/.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
# Rust and maturin build products
/target/
**/dist/
/models-standard/build/
/radixor/models/
*.so
*.egg-info/
# Standard-model payload is generated deterministically below the repository
# build directory and belongs only in the published wheel/sdist.
/models-standard/radixor_models_standard/manifest.json
/models-standard/radixor_models_standard/models/*.rxc
/models-standard/radixor_models_standard/notices/*/NOTICE-model-data.txt
# Local Python environments and caches
.venv/
__pycache__/
*.py[cod]
.pytest_cache/
# Reports produced by direct benchmark-script runs
/benchmarks/results*.csv
/benchmarks/results*.json

254
python/Cargo.lock generated Normal file
View File

@@ -0,0 +1,254 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "indoc"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
dependencies = [
"rustversion",
]
[[package]]
name = "libc"
version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "memoffset"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "portable-atomic"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
[[package]]
name = "proc-macro2"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "pyo3"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
dependencies = [
"cfg-if",
"indoc",
"libc",
"memoffset",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
"unindent",
]
[[package]]
name = "pyo3-build-config"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
dependencies = [
"once_cell",
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
dependencies = [
"libc",
"pyo3-build-config",
]
[[package]]
name = "pyo3-macros"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
"syn",
]
[[package]]
name = "pyo3-macros-backend"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
dependencies = [
"heck",
"proc-macro2",
"pyo3-build-config",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
[[package]]
name = "radixor"
version = "0.0.0"
dependencies = [
"flate2",
"pyo3",
"unicode-general-category",
"unicode-normalization",
]
[[package]]
name = "rustversion"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "simd-adler32"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
[[package]]
name = "syn"
version = "2.0.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "target-lexicon"
version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tinyvec"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicode-general-category"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7"
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]]
name = "unindent"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

20
python/Cargo.toml Normal file
View File

@@ -0,0 +1,20 @@
[package]
name = "radixor"
version = "0.0.0"
edition = "2021"
[lib]
name = "_radixor"
crate-type = ["cdylib"]
[dependencies]
pyo3 = { version = "0.22", features = ["extension-module", "abi3-py39"] }
flate2 = "1.0"
unicode-normalization = "0.1"
unicode-general-category = "0.6"
[profile.release]
lto = true
codegen-units = 1
opt-level = 3
strip = true

28
python/LICENSE Normal file
View File

@@ -0,0 +1,28 @@
Copyright (C) 2026, Leo Galambos
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

253
python/README.md Normal file
View File

@@ -0,0 +1,253 @@
# radixor — Fastest Stemming for Python
**radixor** is a Python extension for the [Radixor](https://github.com/leogalambos/Radixor) stemmer library, built on a Rust core via [PyO3](https://pyo3.rs/). It provides sub-microsecond per-word stemming with a batch API that amortises the Python↔Rust bridge overhead across thousands of words at once.
## Why radixor?
| Library | Approach | Batch API |
|---|---|---|
| **radixor** | Compiled patch-command trie in Rust | ✅ `stem_batch()` |
| PyStemmer (Snowball) | C extension (`libstemmer`) | ✅ `stemWords()` |
| snowballstemmer | Pure-Python Snowball | ✅ (Python loop) |
| NLTK Porter / CISTEM | Pure Python | ❌ |
**Performance.** On the shared UniMorph gold-standard corpus, measuring runtime
stemming only (construction excluded) and with a fair, cache-disabled,
same-input methodology, radixor won all **18 / 18** direct comparisons with
PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run.
At batch size 100, the geometric-mean speedup was **1.67×**. The complete
machine metadata and current results are in the [Python performance
documentation](../docs/python/performance.md); benchmark implementation and
fairness notes are in [`benchmarks/`](benchmarks/README.md).
## Installation
From PyPI, once publication is enabled:
```bash
python -m pip install --only-binary=:all: radixor
```
The GitHub Releases-backed index is the independent alternative:
```bash
python -m pip install --only-binary=:all: \
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
```
The GitHub command becomes usable after the first model and native releases
populate that index. See the [installation guide](../docs/python/installation.md)
for current availability and source-checkout builds.
Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install
also resolves the mandatory pure `radixor-models-standard` dependency
with 20 precompiled standard models. Building the native source distribution
requires Rust ≥ 1.75 and [maturin](https://www.maturin.rs/).
## Quick start
```python
from radixor import Stemmer
s = Stemmer("en") # English (us-uk-default model)
s.stem("running") # → "run"
s.stem("cats") # → "cat"
s.stem("unknown_word") # → None
```
## Batch API — the fast path
```python
words = ["running", "cats", "stemming", "quickly"]
# Amortises the Python→Rust bridge cost across all words at once
stems = s.stem_batch(words)
# → ["run", "cat", "stem", "quick"]
```
For large corpora (tens of thousands of words) the batch call is the recommended interface. It avoids per-call Python frame overhead and keeps the hot loop entirely inside Rust.
## Migrating from PyStemmer
Radixor provides PyStemmer's `stemWord` and `stemWords` method names. These
compatibility methods also follow PyStemmer's fallback behavior: when the trie
has no patch command, they return the original word instead of `None`.
```python
# PyStemmer: import Stemmer
import radixor as Stemmer
stemmer = Stemmer.Stemmer("english")
stemmer.stemWord("running") # → "run"
stemmer.stemWord("unknown_word") # → "unknown_word"
stemmer.stemWords(["running", "unknown"]) # → ["run", "unknown"]
```
The original Radixor methods remain unchanged: `stem` and `stem_batch` return
`None` for words without a matching patch command. Radixor accepts PyStemmer's
full language names for the languages represented by its bundled models, as
well as its existing two-letter codes and model IDs.
## Supported languages
| Code | Language | Model ID |
|---|---|---|
| `cs` | Czech | `cs-cz-default` |
| `da` | Danish | `da-dk-default` |
| `de` | German | `de-de-default` |
| `en` | English | `us-uk-default` |
| `es` | Spanish | `es-es-default` |
| `fa` | Persian | `fa-ir-default` |
| `fi` | Finnish | `fi-fi-default` |
| `fr` | French | `fr-fr-default` |
| `he` | Hebrew | `he-il-default` |
| `hu` | Hungarian | `hu-hu-default` |
| `it` | Italian | `it-it-default` |
| `nb` | Norwegian Bokmål | `nb-no-default` |
| `nl` | Dutch | `nl-nl-default` |
| `nn` | Norwegian Nynorsk | `nn-no-default` |
| `pl` | Polish | `pl-pl-unimorph` |
| `pt` | Portuguese | `pt-pt-default` |
| `ru` | Russian | `ru-ru-default` |
| `sv` | Swedish | `sv-se-default` |
| `uk` | Ukrainian | `uk-ua-default` |
| `yi` | Yiddish | `yi-default` |
## API reference
### `Stemmer(language=None, *, path=None, compiled=None, backward=None, store_original=True, lowercase=True, cache_size=10_000)`
Create a stemmer for the given language code, model ID, custom textual
dictionary, or previously compiled version 7 trie. Textual dictionaries are
compiled in Rust; `compiled=` loads a prepared binary directly.
```python
s = Stemmer("de") # by language code
s = Stemmer("de-de-default") # by model ID
s = Stemmer(path="/data/custom.gz") # custom gzipped dictionary
s = Stemmer(compiled="/data/custom.rxc") # prepared v7 binary
```
`backward` selects the traversal direction; when left as `None` it is derived
from the language (BACKWARD, except right-to-left `fa`/`he`/`yi` which use
FORWARD). `store_original` (default `True`) maps each canonical stem to a no-op
patch so the stem itself is recognised. `lowercase=False` skips runtime
lowercasing for already-normalized input, and `cache_size` enables the bounded
result cache. The default holds up to 10,000 entries, matching PyStemmer;
`cache_size=0` disables it. One cache is shared by `stem()`, `stemWord()`,
`stem_batch()`, and `stemWords()`; the `stem_all*()` methods are not cached.
### `stem(word: str) → str | None`
Return the stem, or `None` when the compiled trie finds no applicable patch
command. This does not mean that lookup is restricted to exact training words.
### `stem_batch(words: list[str]) → list[str | None]`
Stem an entire list. Preferred for large inputs.
### `stemWord(word: str) → str`
PyStemmer-compatible scalar method. Return the original word if it cannot be
stemmed.
### `stemWords(words: list[str]) → list[str]`
PyStemmer-compatible batch method. Return each unrecognized word unchanged.
### `stem_all(word: str) → list[str]`
Return all stems ordered by descending corpus frequency. Useful when multiple valid stems exist.
### `stem_all_batch(words: list[str]) → list[list[str]]`
Return all stems for each word in a batch.
## Compiling a model (compile once, load instantly)
Compiling the trie from a textual dictionary takes time for large languages
(seconds). You can compile it **once** to Radixor's binary format and then load
it near-instantly — the same workflow Java users have:
```python
import radixor
radixor.compile("stemmer.gz", "en.rxc", language="en") # or backward=True/False
s = radixor.Stemmer(compiled="en.rxc") # instant load, no re-compile
```
The compiled file uses Radixor's **v7 trie format and is byte-compatible with
the Java `StemmerPatchTrieBinaryIO`** (the inner stream is identical), so a file
compiled by Java can be loaded by Python and vice versa. `Stemmer(path=...)`
auto-detects whether it was given a compiled trie or a textual dictionary.
## Using a custom model
Provide your own gzipped source dictionary (tab-separated
`stem<TAB>variant1<TAB>variant2…` per line, `#` / `//` line remarks allowed) and
load it directly:
```python
s = Stemmer(path="my_dictionary.gz") # BACKWARD by default
s = Stemmer(path="my_rtl_dictionary.gz", backward=False) # right-to-left
```
The dictionary is compiled to a patch-command trie in Rust at construction time.
## Building from source
```bash
cd Radixor/
pip install maturin build setuptools wheel pytest
./gradlew pythonBuildStandardModels
pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
cd python/
maturin develop --release # editable install with release optimisations
```
From the repository root, Gradle builds the native wheel/sdist and pure
standard-model wheel/sdist without installing them globally:
```bash
./gradlew pythonBuild
```
The convenience tasks `pythonBuildLinux`, `pythonBuildWindows`, and
`pythonBuildMacos` use the host build when the requested platform matches the
current system. Other platforms are cross-compiled with the corresponding Rust
target and therefore require that target and its linker/SDK to be installed.
Override a default target with, for example,
`-PpythonWindowsTarget=x86_64-pc-windows-gnu`. Build artifacts are written below
`build/python/dist/`.
The complete batch benchmark runs Radixor for all bundled languages and every
available comparison engine for the languages it supports, using batch sizes
10, 20, 50, and 100:
Comparison engines are auto-detected in the environment of `pythonExecutable`.
Install `python/benchmarks/requirements-bench.txt` there to enable the complete
comparison set.
```bash
./gradlew pythonBenchmarkAllLanguagesBatch
```
Use `pythonBenchmarkWords`, `pythonBenchmarkRepeats`, and
`pythonBenchmarkWarmup` Gradle properties to tune the run. CSV and JSON reports
are written below `build/reports/python-benchmarks/`.
Neither runtime distribution contains textual dictionaries. The standard data
sdist contains build-ready gzip v7 `.rxc` files, a checksummed provenance
manifest, and per-model CC BY-SA 3.0 notices. They are generated below `build/`
from canonical `models/*/src/modelInput/stemmer.gz` inputs and are never stored
in Git. `./gradlew regeneratePythonStandardModels` performs this deterministic
generation; repository topology selects the 20 defaults and excludes optional
`pl-pl-polimorf`.
`radixor` requires `radixor-models-standard>=1.0,<2.0`. The Python distribution
version is independent of its `2026.1` Java model-catalog identity and of the
individual model versions recorded in the manifest.
## License
The native/API package is BSD-3-Clause — see [LICENSE](LICENSE). Model data
is separately licensed under CC BY-SA 3.0 in its packaged notices.

111
python/benchmarks/README.md Normal file
View File

@@ -0,0 +1,111 @@
# Runtime stemming benchmarks
These scripts measure **runtime stemming throughput only** — model construction
and dictionary compilation happen once during setup and are **excluded** from
every timing. Anyone can reproduce the numbers on their own machine.
## What is measured, and why batch sizes
Each engine is driven through its batch entry point over a fixed word budget
(default 5 000 tokens), split into batches of **10, 20, 50, 100** words. The
harness fits the descriptive line `per_call(N) ≈ intercept + N · slope` across
those sizes. The fit is unconstrained and timing noise can make its intercept
negative, so it describes observed scaling rather than physically separating
overhead from word work. We report the *best* (minimum) of many repeats — the
microbenchmark convention that suppresses OS/GC noise.
## Data — identical to the Java JMH benchmarks
The workload is the **changed-token corpus** built from the repository's
canonical dictionaries (`models/<model>/src/modelInput/stemmer.gz`),
mirroring `LanguageBenchmarkCorpus` in the Java project: each dictionary field
is paired with its line's root, normalized `trim().lower()`, and only tokens
that **differ** from their root are kept, in dictionary order, padded to ≥ 5 000
tokens. See `corpus.py`.
## Fairness — three things that quietly break stemmer comparisons
Getting a *fair* comparison turned out to matter more than any micro-optimization.
Three asymmetries, if left in, make the numbers meaningless:
1. **Result caching.** PyStemmer caches results by default (`maxCacheSize=10000`).
Because a benchmark stems the same corpus every repeat, that cache turns
measured passes into dict lookups rather than stemming. **The harness
explicitly disables both caches**: PyStemmer uses `maxCacheSize=0` and
radixor uses `cache_size=0`, so both engines do real stemming.
snowballstemmer-pure, nltk-porter, and cistem have no cache.
2. **Lowercasing.** Snowball/PyStemmer do no case handling — they assume the
caller pre-lowercased the input (our corpus is pre-lowercased for everyone).
radixor normally lowercases internally; for a same-work comparison the
harness runs radixor with **`lowercase=False`** (assume-already-lowercased),
so Snowball and radixor do identical normalization work on identical input.
CISTEM always performs its own lowercasing and German umlaut normalization;
that unavoidable extra work modestly biases its comparison in radixor's
favour.
3. **Delegation.** `snowballstemmer` delegates to PyStemmer when it is installed
(they become the same C code). The harness bypasses that and uses
snowballstemmer's genuine pure-Python backend, and records each engine's
backing module + whether it is a compiled extension (`--json`) as proof.
## Engines compared
| Engine | Implementation | Batch API |
|---|---|---|
| `radixor` | Rust patch-command trie (cache disabled) | `stem_batch` — one FFI call per batch |
| `PyStemmer` | Snowball C `libstemmer` (cache disabled) | `stemWords(list)` — one C call per batch |
| `snowballstemmer-pure` | Official **pure-Python** Snowball | `stemWords(list)` — Python loop |
| `nltk-porter` | Porter (English), pure Python | scalar loop |
| `cistem` | CISTEM (German), pure Python (`nltk`) | scalar loop |
## Reproduce
```bash
cd python/
maturin develop --release # build the radixor extension into your env
cd ..
./gradlew pythonBuildStandardModels # generate and package standard models
pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
cd python/
pip install -r benchmarks/requirements-bench.txt
python benchmarks/run_benchmark.py --language en de fr ru fi \
--sizes 10 20 50 100 --repeats 21 \
--json benchmarks/results.json --csv benchmarks/results.csv
```
From the repository root, the Gradle integration builds an isolated host wheel
and benchmarks Radixor plus every available comparison engine over all
supported languages with the fixed 10/20/50/100 size sweep:
```bash
./gradlew pythonBenchmarkAllLanguagesBatch
```
The generated CSV and JSON reports are placed in
`build/reports/python-benchmarks/`.
Comparison engines are auto-detected in the environment selected by the Gradle
`pythonExecutable` property. Install `requirements-bench.txt` in that environment
to enable the complete comparison set.
The run prints machine/Python/engine versions and each engine's backing module,
and writes per-point rows (CSV) plus the full report incl. environment and
provenance (JSON), so results are self-describing and verifiable.
## Published results
The canonical, current single-machine results and complete environment metadata
are published on the documentation site's [Python performance
page](../../docs/python/performance.md). Keeping the measured table in one place
prevents results from different CPUs or benchmark runs from being mixed.
## Interpretation
- In the published 2026-08-08 run, **radixor is the fastest stemmer measured in
Python** in all 18 languages directly shared with PyStemmer.
- The benchmark intentionally disables caches. Cached-operation performance is
outside this suite and must not be inferred from its results.
- radixor and Snowball remain different *classes* of stemmer: radixor is
dictionary-based (UniMorph gold coverage), Snowball is rule-based. radixor
gives dictionary-quality stems *and* the best measured throughput.

103
python/benchmarks/corpus.py Normal file
View File

@@ -0,0 +1,103 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Deterministic benchmark corpus, mirroring the Java JMH LanguageBenchmarkCorpus.
The timing workload contains dictionary forms that differ from their canonical
root. Each field is normalized with ``strip().lower()``, retained when it needs
stemming, and repeated in stable dictionary order until the corpus reaches
``MINIMUM_TIMING_TOKEN_COUNT``. This mirrors
``LanguageBenchmarkCorpus.buildChangedTimingCorpus`` and supplies the same
token sequence to every benchmark engine.
"""
from __future__ import annotations
import gzip
from pathlib import Path
MINIMUM_TIMING_TOKEN_COUNT = 5_000
def _normalize(token: str) -> str:
# Java: token.trim().toLowerCase(Locale.ROOT)
return token.strip().lower()
def _contains_whitespace(token: str) -> bool:
return any(ch.isspace() for ch in token)
def read_changed_tokens(dict_gz_path: str | Path) -> list[str]:
"""Return the changed-token list (token != root) in dictionary order.
Not yet padded to the timing minimum; see :func:`build_timing_corpus`.
"""
tokens: list[str] = []
with gzip.open(dict_gz_path, "rt", encoding="utf-8") as fh:
for line in fh:
if not line or line.isspace():
continue
# Match the Java benchmark: only a leading marker starts a comment;
# inline markers remain part of the dictionary field.
if line.startswith("#") or line.startswith("//"):
continue
fields = line.split("\t")
if not fields:
continue
root = _normalize(fields[0])
if not root or _contains_whitespace(root):
continue
for field in fields:
token = _normalize(field)
if not token or _contains_whitespace(token):
continue
if token != root: # changed-token workload
tokens.append(token)
return tokens
def build_timing_corpus(
dict_gz_path: str | Path,
minimum_token_count: int = MINIMUM_TIMING_TOKEN_COUNT,
) -> list[str]:
"""Return the padded changed-token timing corpus (>= minimum_token_count)."""
changed = read_changed_tokens(dict_gz_path)
if not changed:
raise ValueError(f"No changed-token corpus tokens available in {dict_gz_path}")
if len(changed) >= minimum_token_count:
return changed
# Repeat in stable order to reach the minimum, exactly like the Java code.
out: list[str] = []
n = len(changed)
for i in range(minimum_token_count):
out.append(changed[i % n])
return out

360
python/benchmarks/engines.py Normal file
View File

@@ -0,0 +1,360 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Stemmer engine adapters for the benchmark.
Every engine exposes a uniform interface:
engine.name -> str
engine.kind -> "native-batch" | "c-batch" | "py-batch" | "py-loop"
engine.supports(code) -> bool (ISO-639-1 language code)
engine.make(code) -> callable(list[str]) -> list[str] (the batch fn)
``kind`` records how batching is implemented:
- ``native-batch``: one native batch call
- ``c-batch``: C extension with a list entry point
- ``py-batch``: pure-Python object with a list method
- ``py-loop``: repeated scalar calls in Python
Engines whose package is not installed are simply reported as unavailable, so
the benchmark runs with whatever the user has.
"""
from __future__ import annotations
import inspect
from pathlib import Path
from typing import Callable, Optional
def _module_info(obj) -> dict:
"""Provenance of the module that actually provides ``obj``.
The returned metadata identifies the backing module and distinguishes
native extensions from pure-Python implementations.
Resolves via ``type(obj).__module__`` -> ``sys.modules`` because
``inspect.getmodule`` returns ``None`` for Cython extension instances.
"""
import sys
name = type(obj).__module__
mod = sys.modules.get(name) or inspect.getmodule(obj)
file = getattr(mod, "__file__", None) or ""
lower = file.lower()
compiled = (
lower.endswith((".pyd", ".so", ".dll")) or "cpython" in lower or "abi3" in lower
)
return {
"backing_module": name,
"backing_file": file,
"compiled_extension": compiled,
}
# ISO-639-1 -> Snowball algorithm name (matches the Java SnowballLanguageCase
# mapping; nb/nn both map to the single Snowball "norwegian" algorithm).
_SNOWBALL_NAMES: dict[str, str] = {
"cs": "czech",
"fa": "persian",
"pl": "polish",
"da": "danish",
"nl": "dutch",
"en": "english",
"fi": "finnish",
"fr": "french",
"de": "german",
"hu": "hungarian",
"it": "italian",
"nb": "norwegian",
"nn": "norwegian",
"pt": "portuguese",
"ru": "russian",
"es": "spanish",
"sv": "swedish",
"yi": "yiddish",
}
BatchFn = Callable[[list[str]], list[str]]
class Engine:
name: str = "engine"
kind: str = "py-loop"
def available(self) -> bool:
raise NotImplementedError
def supports(self, code: str) -> bool:
raise NotImplementedError
def make(self, code: str) -> BatchFn:
raise NotImplementedError
def provenance(self, code: str) -> dict:
return {
"backing_module": self.name,
"backing_file": "",
"compiled_extension": False,
"algorithm": None,
}
class RadixorEngine(Engine):
name = "radixor"
kind = "native-batch"
def __init__(self, lowercase: bool = False) -> None:
# The shared corpus is already lowercase. Disable Radixor's redundant
# normalization unless a benchmark explicitly includes that cost.
self._lowercase = lowercase
def available(self) -> bool:
try:
import radixor # noqa: F401
return True
except Exception:
return False
def supports(self, code: str) -> bool:
try:
from radixor import _LANGUAGE_ALIASES
return code in _LANGUAGE_ALIASES
except Exception:
return False
def make(self, code: str) -> BatchFn:
from radixor import Stemmer
s = Stemmer(code, lowercase=self._lowercase, cache_size=0)
return s.stem_batch
def provenance(self, code: str) -> dict:
from radixor import Stemmer
s = Stemmer(code, lowercase=self._lowercase, cache_size=0)
info = _module_info(s._core)
info["algorithm"] = "radixor-trie"
info["lowercase"] = self._lowercase
info["cache_disabled"] = True
return info
class PyStemmerEngine(Engine):
name = "PyStemmer"
kind = "c-batch"
def available(self) -> bool:
try:
import Stemmer # noqa: F401
return True
except Exception:
return False
def _algorithms(self) -> set[str]:
import Stemmer
return {a.lower() for a in Stemmer.algorithms()}
def supports(self, code: str) -> bool:
name = _SNOWBALL_NAMES.get(code)
return bool(name) and name in self._algorithms()
@staticmethod
def _new_stemmer(code: str):
import Stemmer
stemmer = Stemmer.Stemmer(_SNOWBALL_NAMES[code])
# Repeated passes would otherwise measure PyStemmer's default cache
# after the first pass. Both native engines therefore run uncached.
stemmer.maxCacheSize = 0
return stemmer
def make(self, code: str) -> BatchFn:
stemmer = self._new_stemmer(code)
# PyStemmer's native list entry point: one C call for the whole batch.
return stemmer.stemWords
def provenance(self, code: str) -> dict:
stemmer = self._new_stemmer(code)
info = _module_info(stemmer)
info["algorithm"] = _SNOWBALL_NAMES[code]
info["cache_disabled"] = True
# Record whether PyStemmer resolves to an independent native extension.
try:
import snowballstemmer
snowball_dir = str(Path(snowballstemmer.__file__).resolve().parent).lower()
except Exception:
snowball_dir = None
backing = info["backing_file"].lower()
info["independent_of_snowballstemmer"] = bool(
info["compiled_extension"]
and (snowball_dir is None or snowball_dir not in backing)
)
return info
class SnowballStemmerEngine(Engine):
"""Pure-Python Snowball backend.
``snowballstemmer.stemmer()`` delegates to PyStemmer (the C extension) when
PyStemmer is installed, which would make this engine a duplicate of
``PyStemmer``. To retain a distinct implementation, this adapter imports
the language's pure-Python class directly from
``snowballstemmer.<name>_stemmer``.
"""
name = "snowballstemmer-pure"
kind = "py-batch"
def available(self) -> bool:
try:
import snowballstemmer # noqa: F401
return True
except Exception:
return False
def _load_class(self, code: str):
import importlib
name = _SNOWBALL_NAMES.get(code)
if not name:
return None
module = importlib.import_module(f"snowballstemmer.{name}_stemmer")
class_name = name.capitalize() + "Stemmer"
return getattr(module, class_name, None)
def supports(self, code: str) -> bool:
try:
return self._load_class(code) is not None
except Exception:
return False
def make(self, code: str) -> BatchFn:
stemmer = self._load_class(code)()
return stemmer.stemWords # pure-Python loop over the list, internally
def provenance(self, code: str) -> dict:
stemmer = self._load_class(code)()
info = _module_info(stemmer)
info["algorithm"] = _SNOWBALL_NAMES[code]
return info
class NltkPorterEngine(Engine):
name = "nltk-porter"
kind = "py-loop"
def available(self) -> bool:
try:
from nltk.stem import PorterStemmer # noqa: F401
return True
except Exception:
return False
def supports(self, code: str) -> bool:
return code == "en" # Porter is English-only
def make(self, code: str) -> BatchFn:
from nltk.stem import PorterStemmer
ps = PorterStemmer()
stem = ps.stem
def batch(words: list[str]) -> list[str]:
return [stem(w) for w in words]
return batch
def provenance(self, code: str) -> dict:
from nltk.stem import PorterStemmer
info = _module_info(PorterStemmer())
info["algorithm"] = "porter"
return info
class CistemEngine(Engine):
"""CISTEM — a fast lightweight German stemmer (German only)."""
name = "cistem"
kind = "py-loop"
def available(self) -> bool:
try:
from nltk.stem.cistem import Cistem # noqa: F401
return True
except Exception:
return False
def supports(self, code: str) -> bool:
return code == "de"
def make(self, code: str) -> BatchFn:
from nltk.stem.cistem import Cistem
stem = Cistem().stem
def batch(words: list[str]) -> list[str]:
return [stem(w) for w in words]
return batch
def provenance(self, code: str) -> dict:
from nltk.stem.cistem import Cistem
info = _module_info(Cistem())
info["algorithm"] = "cistem"
return info
ALL_ENGINES: list[Engine] = [
RadixorEngine(),
PyStemmerEngine(),
SnowballStemmerEngine(),
NltkPorterEngine(),
CistemEngine(),
]
def available_engines(names: Optional[set[str]] = None) -> list[Engine]:
engines = [e for e in ALL_ENGINES if e.available()]
if names:
engines = [e for e in engines if e.name in names]
return engines

6
python/benchmarks/requirements-bench.txt Normal file
View File

@@ -0,0 +1,6 @@
# Optional comparison engines for the runtime stemming benchmark.
# radixor itself must already be built (maturin develop) in the same env.
# Any subset may be installed; the harness auto-detects what is present.
PyStemmer # Snowball as a C extension (libstemmer); real batch API
snowballstemmer # Official pure-Python Snowball (used via its pure backend)
nltk # Porter (English) pure-Python reference baseline

436
python/benchmarks/run_benchmark.py Normal file
View File

@@ -0,0 +1,436 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Runtime stemming benchmark for the radixor Python extension vs alternatives.
Measures ONLY runtime stemming throughput — model construction / dictionary
compilation happens once in setup and is excluded from all timings.
Batch sizes are swept (default 10/20/50/100) and an unconstrained descriptive
line is fitted for each engine: per_call_time(N) = intercept + slope * N. The
fit summarizes scaling across the measured sizes; timing noise can make its
intercept negative, so it must not be read as a physical overhead measurement.
Data is the same as the Java JMH benchmarks: the changed-token corpus derived
from the bundled UniMorph gold-standard dictionaries (see corpus.py).
Examples
--------
python run_benchmark.py --language en
python run_benchmark.py --all-languages --engines radixor
python run_benchmark.py --language en de ru --repeats 15 --csv results.csv
python run_benchmark.py --language en --sizes 10 20 50 100 200 --json out.json
"""
from __future__ import annotations
import argparse
import csv as csvmod
import gc
import json
import platform
import statistics
import sys
import time
from pathlib import Path
from typing import Optional
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE)) # allow running as a plain script
import corpus as corpus_mod # noqa: E402
import engines as engines_mod # noqa: E402
def _chunks(seq: list[str], n: int) -> list[list[str]]:
return [seq[i : i + n] for i in range(0, len(seq), n)]
def _time_sequence_ns(batch_fn, batches: list[list[str]]) -> int:
"""Time one full pass over all batches (nanoseconds)."""
start = time.perf_counter_ns()
for b in batches:
batch_fn(b)
return time.perf_counter_ns() - start
def _linfit(xs: list[float], ys: list[float]) -> tuple[float, float]:
"""Ordinary least squares: returns (intercept, slope)."""
n = len(xs)
mean_x = sum(xs) / n
mean_y = sum(ys) / n
sxx = sum((x - mean_x) ** 2 for x in xs)
sxy = sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys))
slope = sxy / sxx if sxx else 0.0
intercept = mean_y - slope * mean_x
return intercept, slope
_DIST_NAMES = {
"radixor": "radixor",
"PyStemmer": "PyStemmer",
"snowballstemmer-pure": "snowballstemmer",
"nltk-porter": "nltk",
}
def _engine_version(name: str) -> Optional[str]:
import importlib.metadata as md
dist = _DIST_NAMES.get(name)
if not dist:
return None
try:
return md.version(dist)
except Exception:
return "editable" if name == "radixor" else None
def _processor_name() -> str:
"""Return a useful CPU model name without adding a platform dependency."""
name = platform.processor().strip()
if name:
return name
cpuinfo = Path("/proc/cpuinfo")
if cpuinfo.is_file():
for line in cpuinfo.read_text(encoding="utf-8", errors="replace").splitlines():
key, separator, value = line.partition(":")
if separator and key.strip() in {"model name", "Hardware"}:
name = value.strip()
if name:
return name
return "unknown"
def run(args) -> dict:
from radixor import _LANGUAGE_ALIASES
engine_filter = set(args.engines) if args.engines else None
engines = engines_mod.available_engines(engine_filter)
if not engines:
print(
"No stemmer engines available. Install PyStemmer / snowballstemmer / nltk.",
file=sys.stderr,
)
sys.exit(2)
results: list[dict] = []
strict_engine_names = (
(engine_filter or {"radixor"}) if args.all_languages else set()
)
failures: list[str] = []
available_engine_names = {engine.name for engine in engines}
for missing_engine in sorted(strict_engine_names - available_engine_names):
failures.append(f"engine unavailable: {missing_engine}")
for code in args.language:
model_id = _LANGUAGE_ALIASES.get(code, code)
if args.model_path:
dict_path = Path(args.model_path)
else:
# Corpus construction deliberately uses the canonical repository
# source. Runtime distributions contain only compiled model data.
dict_path = (
HERE.parents[1]
/ "models"
/ model_id
/ "src"
/ "modelInput"
/ "stemmer.gz"
)
if not dict_path.is_file():
print(
f"[{code}] canonical benchmark dictionary not found: {dict_path}",
file=sys.stderr,
)
if args.all_languages:
failures.append(f"{code}: dictionary not found: {dict_path}")
continue
full = corpus_mod.build_timing_corpus(dict_path)
budget = min(args.words, len(full)) if args.words > 0 else len(full)
pool = full[:budget]
print(
f"\n=== language={code} model={model_id} "
f"corpus={len(pool)} changed tokens ==="
)
for engine in engines:
if not engine.supports(code):
if engine.name in strict_engine_names:
failures.append(f"{code}: engine does not support {engine.name}")
continue
try:
batch_fn = engine.make(code)
except Exception as exc: # pragma: no cover - engine setup failure
print(f" [{engine.name}] setup failed: {exc}", file=sys.stderr)
if engine.name in strict_engine_names:
failures.append(f"{code}: {engine.name} setup failed: {exc}")
continue
prov = engine.provenance(code)
results.append(
{
"language": code,
"model": model_id,
"engine": engine.name,
"kind": engine.kind,
"batch_size": "PROVENANCE",
**prov,
}
)
print(
f" {engine.name:<16} backing={prov.get('backing_module')} "
f"compiled={prov.get('compiled_extension')} "
f"algo={prov.get('algorithm')}"
)
# sanity: output length must equal input length
probe = batch_fn(pool[: min(8, len(pool))])
if len(probe) != min(8, len(pool)):
print(
f" [{engine.name}] unexpected output shape; skipping",
file=sys.stderr,
)
if engine.name in strict_engine_names:
failures.append(
f"{code}: {engine.name} returned an unexpected output shape"
)
continue
per_call_best: list[float] = []
for size in args.sizes:
batches = _chunks(pool, size)
n_calls = len(batches)
n_words = len(pool)
# warmup
for _ in range(args.warmup):
_time_sequence_ns(batch_fn, batches)
gc_was_enabled = gc.isenabled()
gc.disable()
try:
totals = [
_time_sequence_ns(batch_fn, batches)
for _ in range(args.repeats)
]
finally:
if gc_was_enabled:
gc.enable()
med_total = statistics.median(totals)
min_total = min(totals)
# Per-word/per-call reported from the best (min) pass — the
# microbenchmark convention that suppresses OS/GC scheduling
# noise. The later OLS fit is descriptive and unconstrained.
per_word_ns = min_total / n_words
per_call_ns = min_total / n_calls
per_call_best.append(per_call_ns)
throughput = n_words / (min_total / 1e9)
row = {
"language": code,
"model": model_id,
"engine": engine.name,
"kind": engine.kind,
"batch_size": size,
"calls": n_calls,
"words": n_words,
"repeats": args.repeats,
"median_total_ms": med_total / 1e6,
"min_total_ms": min_total / 1e6,
"per_word_ns": per_word_ns,
"per_call_us": per_call_ns / 1e3,
"throughput_words_per_s": throughput,
}
results.append(row)
print(
f" {engine.name:<16} [{engine.kind:<12}] "
f"N={size:<4} {per_word_ns:8.1f} ns/word "
f"{per_call_ns / 1e3:8.2f} us/call "
f"{throughput / 1e6:6.2f} M words/s"
)
# Unconstrained descriptive OLS fit across batch sizes. Keep the
# historical JSON key for report compatibility.
if len(args.sizes) >= 2:
intercept_ns, slope_ns = _linfit(
[float(s) for s in args.sizes], per_call_best
)
results.append(
{
"language": code,
"model": model_id,
"engine": engine.name,
"kind": engine.kind,
"batch_size": "FIT",
"regie_ns_per_call": intercept_ns,
"real_ns_per_word": slope_ns,
}
)
print(
f" {engine.name:<16} -> estimated intercept/call = "
f"{intercept_ns / 1e3:7.2f} us "
f"estimated slope = {slope_ns:7.1f} ns/word"
)
if args.all_languages:
expected_measurements = {
(code, engine_name, size)
for code in args.language
for engine_name in strict_engine_names
for size in args.sizes
}
actual_measurements = {
(row["language"], row["engine"], row["batch_size"])
for row in results
if isinstance(row.get("batch_size"), int)
}
missing_measurements = sorted(expected_measurements - actual_measurements)
if missing_measurements:
failures.append(f"missing measurement rows: {missing_measurements}")
if failures:
raise RuntimeError(
"Incomplete all-language benchmark: " + "; ".join(failures)
)
return {
"environment": {
"platform": platform.platform(),
"processor": _processor_name(),
"python": sys.version.split()[0],
"python_impl": platform.python_implementation(),
"engine_versions": {e.name: _engine_version(e.name) for e in engines},
},
"parameters": {
"languages": args.language,
"sizes": args.sizes,
"words_budget": args.words,
"repeats": args.repeats,
"warmup": args.warmup,
},
"results": results,
}
def main() -> None:
p = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
languages = p.add_mutually_exclusive_group()
languages.add_argument(
"--language",
"-l",
nargs="+",
default=None,
help="ISO-639-1 language code(s) or model id(s). Default: en",
)
languages.add_argument(
"--all-languages",
action="store_true",
help="Benchmark every language alias bundled by radixor",
)
p.add_argument(
"--sizes",
"-s",
type=int,
nargs="+",
default=[10, 20, 50, 100],
help="Batch sizes to sweep. Default: 10 20 50 100",
)
p.add_argument(
"--words",
"-w",
type=int,
default=5000,
help="Words processed per measurement (<=0 = whole corpus). Default: 5000",
)
p.add_argument(
"--repeats",
"-r",
type=int,
default=15,
help="Timed repeats per point (best/min reported). Default: 15",
)
p.add_argument("--warmup", type=int, default=3, help="Warmup passes. Default: 3")
p.add_argument(
"--engines",
nargs="+",
default=None,
help="Restrict to named engines (radixor PyStemmer snowballstemmer nltk-porter)",
)
p.add_argument(
"--model-path",
default=None,
help="Explicit gzipped dictionary path (single-language runs)",
)
p.add_argument("--csv", default=None, help="Write per-point rows to this CSV file")
p.add_argument(
"--json", default=None, help="Write full results (incl. environment) to JSON"
)
args = p.parse_args()
if args.all_languages:
from radixor import _LANGUAGE_ALIASES
args.language = sorted(_LANGUAGE_ALIASES)
elif args.language is None:
args.language = ["en"]
report = run(args)
if args.json:
Path(args.json).write_text(json.dumps(report, indent=2), encoding="utf-8")
print(f"\nwrote {args.json}")
if args.csv:
rows = [
r
for r in report["results"]
if r.get("batch_size") not in ("FIT", "PROVENANCE")
]
if rows:
with open(args.csv, "w", newline="", encoding="utf-8") as fh:
w = csvmod.DictWriter(fh, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
print(f"wrote {args.csv}")
print("\nEnvironment:")
for k, v in report["environment"].items():
print(f" {k}: {v}")
if __name__ == "__main__":
main()

12
python/models-standard/LICENSE-MODEL-DATA.txt Normal file
View File

@@ -0,0 +1,12 @@
Radixor Standard Model Data License
The model data in this distribution, including Radixor's protectable
selection, transformation, metadata, and packaging contributions, is licensed
under Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0):
https://creativecommons.org/licenses/by-sa/3.0/
Each model's attribution, upstream provenance, and license details are recorded
in radixor_models_standard/notices/<model-id>/NOTICE-model-data.txt. Those
notices form part of this distribution and must be retained with redistributed
model data.

4
python/models-standard/MANIFEST.in Normal file
View File

@@ -0,0 +1,4 @@
include LICENSE-MODEL-DATA.txt
include README.md
recursive-include radixor_models_standard *.json *.rxc *.txt
global-exclude stemmer.gz *.gz __pycache__ *.py[cod]

26
python/models-standard/README.md Normal file
View File

@@ -0,0 +1,26 @@
# Radixor standard models
This pure-Python distribution supplies Radixor's 20 precompiled standard
language models. It is installed automatically by `pip install radixor`; users
normally do not import it directly.
The catalog version is `2026.1`. Individual model versions recorded in the
generated `radixor_models_standard/manifest.json` are currently `1.0.0`. The
optional Polish PoliMorf model is intentionally not part of the standard
catalog.
The first Python model distribution is released as `1.0.0`; its version is
independent of both the catalog identity and the individual model versions.
The checked-in descriptors use `0.0.0` as a deliberate non-release placeholder.
The release workflow creates an isolated project below `build/`, injects the Git
tag version, and deterministically compiles all model resources there.
Only gzip-compressed Radixor v7 (`.rxc`) tries are shipped. They are release
artifacts, not checked-in repository files. Canonical textual dictionaries
remain in the Radixor source repository and are not included in this wheel or
source distribution. Model data is licensed under CC BY-SA 3.0; see
`LICENSE-MODEL-DATA.txt` and the generated per-model notices.
The checked-out directory is intentionally only a packaging skeleton and is
not directly buildable as the complete data distribution. From the repository
root, use `./gradlew pythonBuildStandardModels`; the generated project and its
wheel/sdist are written below `build/python/`.

29
python/models-standard/pyproject.toml Normal file
View File

@@ -0,0 +1,29 @@
[build-system]
requires = ["setuptools>=77"]
build-backend = "setuptools.build_meta"
[project]
name = "radixor-models-standard"
version = "0.0.0"
requires-python = ">=3.9"
description = "Precompiled standard language models for Radixor"
readme = "README.md"
license = "CC-BY-SA-3.0"
license-files = ["LICENSE-MODEL-DATA.txt"]
keywords = ["stemming", "nlp", "linguistics", "model-data"]
classifiers = [
"Programming Language :: Python :: 3",
"Topic :: Text Processing :: Linguistic",
]
[tool.setuptools]
include-package-data = true
[tool.setuptools.packages.find]
where = ["."]
include = ["radixor_models_standard*"]
[tool.setuptools.package-data]
radixor_models_standard = ["manifest.json"]
"radixor_models_standard.models" = ["*.rxc"]
"radixor_models_standard.notices" = ["*/*.txt"]

37
python/models-standard/radixor_models_standard/__init__.py Normal file
View File

@@ -0,0 +1,37 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Installed resources for the Radixor 2026.1 standard model catalog."""
CATALOG_VERSION = "2026.1"
__version__ = "0.0.0"
__all__ = ["CATALOG_VERSION"]

32
python/models-standard/radixor_models_standard/models/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Compiled model resources; not a public Python API."""

32
python/models-standard/radixor_models_standard/notices/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Per-model attribution and license notice resources."""

82
python/pyproject.toml Normal file
View File

@@ -0,0 +1,82 @@
[build-system]
requires = ["maturin>=1.7,<2.0"]
build-backend = "maturin"
[project]
name = "radixor"
version = "0.0.0"
requires-python = ">=3.9"
description = "Radixor stemmer – fastest stemming for Python, backed by Rust"
readme = "README.md"
license = "BSD-3-Clause"
license-files = ["LICENSE"]
dependencies = ["radixor-models-standard>=1.0,<2.0"]
keywords = [
"python",
"information retrieval",
"language processing",
"morphology",
"stemming algorithms",
"stemmers",
"nlp",
"rust",
]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Natural Language :: Czech",
"Natural Language :: Danish",
"Natural Language :: Dutch",
"Natural Language :: English",
"Natural Language :: Finnish",
"Natural Language :: French",
"Natural Language :: German",
"Natural Language :: Hebrew",
"Natural Language :: Hungarian",
"Natural Language :: Italian",
"Natural Language :: Norwegian",
"Natural Language :: Persian",
"Natural Language :: Polish",
"Natural Language :: Portuguese",
"Natural Language :: Russian",
"Natural Language :: Spanish",
"Natural Language :: Swedish",
"Natural Language :: Ukrainian",
"Natural Language :: Yiddish",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Rust",
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
"Topic :: Text Processing :: Indexing",
"Topic :: Text Processing :: Linguistic",
]
[tool.maturin]
python-source = "."
module-name = "radixor._radixor"
features = ["pyo3/extension-module"]
exclude = [
"radixor/models/**",
"models-standard/**",
"dist/**",
"benchmarks/results*.csv",
"benchmarks/results*.json",
"**/__pycache__/**",
"**/*.pyc",
"**/*.pyo",
"**/*.pyd",
]
[tool.ruff]
extend-exclude = ["models-standard/build", "target"]
line-length = 88
target-version = "py39"
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I"]

398
python/radixor/__init__.py Normal file
View File

@@ -0,0 +1,398 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Python API for the Rust-backed Radixor stemmer.
Usage::
from radixor import Stemmer
s = Stemmer("en")
print(s.stem("running")) # single word
print(s.stem_batch(words)) # batch API for collections
"""
from __future__ import annotations
import gzip
import hashlib
import importlib.resources
import json
import re
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Iterator, Optional
from radixor._radixor import StemmerCore
from radixor._radixor import compile as _compile
_LANGUAGE_ALIASES: dict[str, str] = {
# Friendly aliases -> model ID
"cs": "cs-cz-default",
"czech": "cs-cz-default",
"da": "da-dk-default",
"danish": "da-dk-default",
"de": "de-de-default",
"german": "de-de-default",
"en": "us-uk-default",
"english": "us-uk-default",
"es": "es-es-default",
"spanish": "es-es-default",
"fa": "fa-ir-default",
"persian": "fa-ir-default",
"fi": "fi-fi-default",
"finnish": "fi-fi-default",
"fr": "fr-fr-default",
"french": "fr-fr-default",
"he": "he-il-default",
"hebrew": "he-il-default",
"hu": "hu-hu-default",
"hungarian": "hu-hu-default",
"it": "it-it-default",
"italian": "it-it-default",
"nb": "nb-no-default",
"norwegian": "nb-no-default",
"nl": "nl-nl-default",
"dutch": "nl-nl-default",
"nn": "nn-no-default",
"pl": "pl-pl-unimorph",
"polish": "pl-pl-unimorph",
"pt": "pt-pt-default",
"portuguese": "pt-pt-default",
"ru": "ru-ru-default",
"russian": "ru-ru-default",
"sv": "sv-se-default",
"swedish": "sv-se-default",
"uk": "uk-ua-default",
"ukrainian": "uk-ua-default",
"yi": "yi-default",
"yiddish": "yi-default",
}
# Right-to-left languages use FORWARD traversal; everything else BACKWARD.
# Keyed by model ID prefix (language part).
_RIGHT_TO_LEFT_MODELS: frozenset[str] = frozenset(
{"fa-ir-default", "he-il-default", "yi-default"}
)
_STANDARD_PACKAGE = "radixor_models_standard"
_STANDARD_CATALOG_VERSION = "2026.1"
_STANDARD_DISTRIBUTION_VERSION = re.compile(
r"(?:0\.0\.0|1\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))\Z"
)
_MODEL_ID = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*\Z")
_SHA256 = re.compile(r"[0-9a-f]{64}\Z")
_V7_MAGIC = b"EGTR"
_V7_VERSION = 7
def _load_standard_manifest() -> dict[str, Any]:
"""Load and validate the installed standard model catalog manifest."""
try:
ref = importlib.resources.files(_STANDARD_PACKAGE).joinpath("manifest.json")
except (ModuleNotFoundError, TypeError) as exc:
raise ModuleNotFoundError(
"The standard Radixor model package is not installed. Install a compatible "
"provider with 'pip install radixor-models-standard>=1.0,<2.0', "
"or reinstall Radixor with 'pip install radixor'."
) from exc
try:
manifest = json.loads(ref.read_text(encoding="utf-8"))
except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError) as exc:
raise RuntimeError(
"The installed radixor-models-standard manifest is missing or corrupt; "
"reinstall radixor-models-standard."
) from exc
try:
models = manifest["models"]
format_info = manifest["format"]
if manifest["schema_version"] != 1:
raise ValueError("unsupported schema_version")
if manifest["catalog_version"] != _STANDARD_CATALOG_VERSION:
raise ValueError(
f"catalog {manifest['catalog_version']!r} is incompatible with "
f"Radixor catalog {_STANDARD_CATALOG_VERSION!r}"
)
distribution_version = manifest["distribution_version"]
if (
not isinstance(distribution_version, str)
or _STANDARD_DISTRIBUTION_VERSION.fullmatch(distribution_version) is None
):
raise ValueError("incompatible distribution_version")
if format_info != {"compression": "gzip", "magic": "EGTR", "version": 7}:
raise ValueError("unsupported compiled model format")
if not isinstance(models, list) or not models:
raise ValueError("models must be a non-empty list")
seen: set[str] = set()
for model in models:
model_id = model["id"]
if (
not isinstance(model_id, str)
or _MODEL_ID.fullmatch(model_id) is None
or model_id in seen
or model["file"] != f"models/{model_id}.rxc"
or not isinstance(model["version"], str)
or _SHA256.fullmatch(model["sha256"]) is None
):
raise ValueError("invalid model entry")
seen.add(model_id)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"The installed radixor-models-standard manifest is incompatible or corrupt: {exc}. "
"Install radixor-models-standard>=1.0,<2.0."
) from exc
return manifest
def _manifest_model(model_id: str) -> dict[str, Any]:
if not isinstance(model_id, str) or _MODEL_ID.fullmatch(model_id) is None:
raise ValueError(
f"Invalid Radixor model ID {model_id!r}; expected lowercase letters, digits, and hyphens."
)
manifest = _load_standard_manifest()
for model in manifest["models"]:
if model["id"] == model_id:
return model
raise FileNotFoundError(
f"Model '{model_id}' is not in the standard Radixor catalog. "
"Pass a custom source path via Stemmer(path=...) or a compiled v7 path "
"via Stemmer(compiled=...)."
)
def _validate_standard_model(path: Path, model: dict[str, Any]) -> None:
try:
digest = hashlib.sha256(path.read_bytes()).hexdigest()
except OSError as exc:
raise RuntimeError(
f"Standard model '{model['id']}' cannot be read; reinstall radixor-models-standard."
) from exc
if digest != model["sha256"]:
raise RuntimeError(
f"Standard model '{model['id']}' failed SHA-256 validation; "
"reinstall radixor-models-standard."
)
try:
with gzip.open(path, "rb") as stream:
header = stream.read(8)
except (OSError, EOFError) as exc:
raise RuntimeError(
f"Standard model '{model['id']}' is not a valid gzip-compressed v7 resource; "
"reinstall radixor-models-standard."
) from exc
if header[:4] != _V7_MAGIC or len(header) != 8:
raise RuntimeError(
f"Standard model '{model['id']}' does not contain the Radixor EGTR format marker; "
"reinstall radixor-models-standard."
)
version = int.from_bytes(header[4:8], "big", signed=True)
if version != _V7_VERSION:
raise RuntimeError(
f"Standard model '{model['id']}' uses unsupported compiled format v{version}; "
f"Radixor requires v{_V7_VERSION}."
)
@contextmanager
def _standard_model_path(model_id: str) -> Iterator[Path]:
"""Yield a validated standard model path for synchronous native loading."""
model = _manifest_model(model_id)
ref = (
importlib.resources.files(_STANDARD_PACKAGE)
.joinpath("models")
.joinpath(f"{model_id}.rxc")
)
try:
with importlib.resources.as_file(ref) as path:
if not path.is_file():
raise FileNotFoundError
_validate_standard_model(path, model)
yield path
except FileNotFoundError as exc:
raise FileNotFoundError(
f"Standard model '{model_id}' is missing from radixor-models-standard; "
"reinstall radixor-models-standard."
) from exc
def _is_backward(model_id: str) -> bool:
"""Traversal direction implied by the model's language (RTL => FORWARD)."""
return model_id not in _RIGHT_TO_LEFT_MODELS
class Stemmer:
"""Thread-safe stemmer backed by a Radixor patch-command trie.
Standard language models are loaded from validated, precompiled v7 resources
supplied by the mandatory ``radixor-models-standard`` distribution.
Parameters
----------
language:
Two-letter ISO 639-1 code (e.g. ``"en"``) or a full model ID
(e.g. ``"us-uk-default"``). Ignored when ``path`` is given.
path:
Explicit path to either a gzipped source dictionary or a compiled
``.rxc`` trie (Java-interoperable v7 format); the format is
auto-detected. Takes precedence over ``language``.
compiled:
Alias for ``path`` intended for compiled ``.rxc`` files (see
:func:`compile`). For compiled input, ``backward`` / ``store_original``
are baked into the file and ignored.
backward:
Traversal direction override. When ``None`` (default) it is derived
from the language (BACKWARD, except right-to-left fa/he/yi which use
FORWARD). Only consulted for ``path``-based construction if given.
store_original:
When ``True`` (default) each canonical stem maps to the no-op patch,
so the stem itself is recognised.
lowercase:
When ``True`` (default) lookups lowercase the input word. Set to
``False`` when you guarantee the input is already lowercased (skips the
per-lookup normalization; the model's keys are always lowercase).
cache_size:
Maximum entries in the bounded result cache (default ``10_000``,
matching PyStemmer). Set to ``0`` to disable caching. Cached results are
shared by :meth:`stem`, :meth:`stemWord`, :meth:`stem_batch`, and
:meth:`stemWords`; ``stem_all`` methods are not cached.
"""
def __init__(
self,
language: Optional[str] = None,
*,
path: Optional[str] = None,
compiled: Optional[str] = None,
backward: Optional[bool] = None,
store_original: bool = True,
lowercase: bool = True,
cache_size: int = 10_000,
) -> None:
source = path if path is not None else compiled
if source is not None:
model_path = source
is_backward = True if backward is None else backward
elif language is not None:
model_id = _LANGUAGE_ALIASES.get(language, language)
is_backward = _is_backward(model_id) if backward is None else backward
with _standard_model_path(model_id) as model_path:
self._core = StemmerCore(
str(model_path), is_backward, store_original, lowercase, cache_size
)
return
else:
raise ValueError("Provide 'language', 'path', or 'compiled'.")
self._core = StemmerCore(
model_path, is_backward, store_original, lowercase, cache_size
)
def stem(self, word: str) -> Optional[str]:
"""Return a stem, or ``None`` when no patch command applies."""
return self._core.stem(word)
def stem_batch(self, words: list[str]) -> list[Optional[str]]:
"""Stem many words in one call.
Preferred over calling :meth:`stem` in a loop: the Python→Rust bridge
overhead is amortised across the whole batch, making this significantly
faster for large word lists.
Returns a list of the same length; entries are ``None`` when the
compiled trie finds no applicable patch command.
"""
return self._core.stem_batch(words)
def stemWord(self, word: str) -> str:
"""Return a stem using PyStemmer-compatible fallback semantics.
If no patch command can be found, return *word* unchanged. Use
:meth:`stem` when a missing result must remain distinguishable as
``None``.
"""
return self._core.stemWord(word)
def stemWords(self, words: list[str]) -> list[str]:
"""Stem words using PyStemmer-compatible fallback semantics.
The returned list has the same length and order as *words*; each word
without a matching patch command is returned unchanged.
"""
return self._core.stemWords(words)
def stem_all(self, word: str) -> list[str]:
"""Return all stems for *word* ordered by descending frequency."""
return self._core.stem_all(word)
def stem_all_batch(self, words: list[str]) -> list[list[str]]:
"""Return all stems for each word in *words* as a list of lists."""
return self._core.stem_all_batch(words)
def compile(
source: str,
out_path: str,
*,
language: Optional[str] = None,
backward: Optional[bool] = None,
store_original: bool = True,
lowercase: bool = True,
) -> None:
"""Compile a textual source dictionary into a Java-interoperable compiled
trie file (v7 format) that :class:`Stemmer` can load instantly.
Parameters
----------
source:
Path to a gzipped (or plain) TSV source dictionary.
out_path:
Destination compiled file (conventionally ``*.rxc``).
language:
Optional language code/model ID used only to derive ``backward`` when
it is not given (right-to-left fa/he/yi compile FORWARD).
backward:
Traversal direction. When ``None`` it is derived from ``language`` if
provided, otherwise defaults to BACKWARD.
store_original, lowercase:
Same meaning as :class:`Stemmer`; baked into the compiled file.
The resulting file is byte-compatible (inner stream) with the Radixor Java
``StemmerPatchTrieBinaryIO`` v7 format, so Java and Python can share it.
"""
if backward is None:
if language is not None:
backward = _is_backward(_LANGUAGE_ALIASES.get(language, language))
else:
backward = True
_compile(source, out_path, backward, store_original, lowercase)
__all__ = ["Stemmer", "compile"]

0
python/radixor/py.typed Normal file
View File

171
python/scripts/assemble_release.py Executable file
View File

@@ -0,0 +1,171 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Validate and assemble the exact files allowed in a Python GitHub Release."""
from __future__ import annotations
import argparse
import hashlib
import re
import shutil
from pathlib import Path
from verify_distributions import (
_verify_main_sdist,
_verify_main_wheel,
_verify_standard_sdist,
_verify_standard_wheel,
)
REPOSITORY = Path(__file__).resolve().parents[2]
BUILD_ROOT = (REPOSITORY / "build").resolve()
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
def _release_files(root: Path) -> list[Path]:
files = sorted(
path
for path in root.rglob("*")
if path.is_file()
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
)
names = [path.name for path in files]
if len(names) != len(set(names)):
raise ValueError(f"Duplicate release filenames: {names}")
return files
def _require_one(files: list[Path], predicate, description: str) -> Path:
matches = [path for path in files if predicate(path.name)]
if len(matches) != 1:
raise ValueError(
f"Expected one {description}, found {[path.name for path in matches]}"
)
return matches[0]
def _validate_native(files: list[Path], version: str) -> None:
prefix = f"radixor-{version}-cp39-abi3-"
wheels = [path for path in files if path.suffix == ".whl"]
sdist = _require_one(
files, lambda name: name == f"radixor-{version}.tar.gz", "native sdist"
)
if len(wheels) != 4:
raise ValueError(
f"Expected four native wheels, found {[path.name for path in wheels]}"
)
expected = {
"linux-x86_64": lambda name: name.startswith(prefix)
and "manylinux" in name
and name.endswith("x86_64.whl"),
"linux-aarch64": lambda name: name.startswith(prefix)
and "manylinux" in name
and name.endswith("aarch64.whl"),
"macos-universal2": lambda name: name.startswith(prefix)
and "macosx" in name
and name.endswith("universal2.whl"),
"windows-x86_64": lambda name: name == f"{prefix}win_amd64.whl",
}
for description, predicate in expected.items():
_require_one(wheels, predicate, description)
for wheel in wheels:
_verify_main_wheel(wheel, version)
_verify_main_sdist(sdist, version)
def _validate_models(files: list[Path], version: str) -> None:
wheel = _require_one(
files,
lambda name: name == f"radixor_models_standard-{version}-py3-none-any.whl",
"standard-model wheel",
)
sdist = _require_one(
files,
lambda name: name == f"radixor_models_standard-{version}.tar.gz",
"standard-model sdist",
)
if len(files) != 2:
raise ValueError(
f"Unexpected standard-model release files: {[path.name for path in files]}"
)
_verify_standard_wheel(wheel, version)
_verify_standard_sdist(sdist, version)
def _output_directory(path: Path) -> Path:
output = path.resolve()
try:
output.relative_to(BUILD_ROOT)
except ValueError as exc:
raise ValueError(f"Release output must be below {BUILD_ROOT}") from exc
if output == BUILD_ROOT:
raise ValueError("Release output cannot be the build root")
if output.exists():
shutil.rmtree(output)
output.mkdir(parents=True)
return output
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("distribution", choices=("native", "models-standard"))
parser.add_argument("version")
parser.add_argument("artifacts", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
if VERSION.fullmatch(args.version) is None:
raise SystemExit(f"Invalid stable release version: {args.version!r}")
files = _release_files(args.artifacts.resolve())
if args.distribution == "native":
_validate_native(files, args.version)
else:
_validate_models(files, args.version)
output = _output_directory(args.output)
for source in files:
shutil.copy2(source, output / source.name)
checksums = [
f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}"
for path in sorted(output.iterdir())
if path.is_file()
]
(output / "SHA256SUMS").write_text(
"\n".join(checksums) + "\n", encoding="utf-8", newline="\n"
)
print(f"assembled {len(files)} release artifacts in {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

129
python/scripts/build_standard_distribution.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Build the standard-model wheel and sdist through its declared backend."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import tempfile
from pathlib import Path
EXPECTED_MODEL_COUNT = 20
def _ignore_build_artifacts(_directory: str, names: list[str]) -> set[str]:
"""Exclude local build state from the isolated distribution source tree."""
ignored = {
name
for name in names
if name in {"build", "dist", "__pycache__"}
or name.endswith((".egg-info", ".pyc", ".pyo"))
}
return ignored
def _validate_generated_project(project: Path) -> None:
"""Reject an ungenerated or incomplete standard-model project."""
package = project / "radixor_models_standard"
manifest_path = package / "manifest.json"
if not manifest_path.is_file():
raise ValueError(
"standard-model manifest is missing; run build_standard_models.py first"
)
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
model_ids = {
model["id"] for model in manifest["models"] if isinstance(model, dict)
}
except (json.JSONDecodeError, KeyError, TypeError) as exc:
raise ValueError("standard-model manifest is invalid") from exc
compiled = {path.stem for path in (package / "models").glob("*.rxc")}
notices = {
path.parent.name
for path in (package / "notices").glob("*/NOTICE-model-data.txt")
}
if len(model_ids) != EXPECTED_MODEL_COUNT:
raise ValueError(
f"expected {EXPECTED_MODEL_COUNT} standard models, found {len(model_ids)}"
)
if compiled != model_ids:
raise ValueError("compiled standard models do not match the manifest")
if notices != model_ids:
raise ValueError("standard-model notices do not match the manifest")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True, type=Path)
parser.add_argument("--outdir", required=True, type=Path)
args = parser.parse_args()
try:
from setuptools.build_meta import build_sdist, build_wheel
except ImportError as exc:
raise SystemExit(
"setuptools>=77 is required to build radixor-models-standard"
) from exc
source_project = args.project.resolve()
try:
_validate_generated_project(source_project)
except ValueError as exc:
parser.error(str(exc))
output = args.outdir.resolve()
output.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="radixor-models-build-") as temporary:
project = Path(temporary, "project")
shutil.copytree(
source_project,
project,
ignore=_ignore_build_artifacts,
)
os.chdir(project)
wheel = build_wheel(str(output))
sdist = build_sdist(str(output))
print(f"built {wheel} and {sdist}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

285
python/scripts/build_standard_models.py Normal file
View File

@@ -0,0 +1,285 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Regenerate the pure-Python standard model package deterministically.
The canonical build topology selects ``default`` models. Source dictionaries
are compiler inputs only and are never copied into either Python distribution.
Run this script with a built Radixor extension importable by the selected
Python interpreter.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Any
REPOSITORY = Path(__file__).resolve().parents[2]
PYTHON_ROOT = REPOSITORY / "python"
BUILD_ROOT = REPOSITORY / "build"
SOURCE_PROJECT = PYTHON_ROOT / "models-standard"
TOPOLOGY = REPOSITORY / "models" / "model-projects.properties"
CATALOG_VERSION = REPOSITORY / "models" / "catalog-version.txt"
EXPECTED_FORMAT = {"compression": "gzip", "magic": "EGTR", "version": 7}
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
METADATA_FIELDS = (
"sourceName",
"sourceVersion",
"sourceRevision",
"sourceProject",
"sourceRepository",
"sourceDataset",
"sourceRevisionStatus",
"sourceLicense",
"sourceLicenseUri",
"sourceAttribution",
"sourceVerificationDate",
"transformationsSummary",
)
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _default_models() -> list[str]:
entries: dict[str, str] = {}
for raw_line in TOPOLOGY.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
model_id, separator, membership = line.partition("=")
if not separator or not model_id or membership not in {"default", "optional"}:
raise ValueError(f"Invalid model topology line: {raw_line!r}")
entries[model_id] = membership
model_ids = sorted(
model_id for model_id, membership in entries.items() if membership == "default"
)
if len(model_ids) != 20 or "pl-pl-polimorf" in model_ids:
raise ValueError(
"Standard Python catalog must contain 20 defaults and exclude pl-pl-polimorf"
)
return model_ids
def _gradle_metadata(path: Path) -> dict[str, str]:
text = path.read_text(encoding="utf-8")
result: dict[str, str] = {}
for field in METADATA_FIELDS:
match = re.search(rf"^\s*{field}\s*=\s*'([^']*)'\s*$", text, re.MULTILINE)
if match is None:
raise ValueError(f"Missing {field} in {path}")
result[field] = match.group(1)
return result
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
text = path.read_text(encoding="utf-8")
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
if count != 1:
raise ValueError(f"Expected exactly one version field in {path}")
path.write_text(updated, encoding="utf-8", newline="\n")
def _stage_project(output: Path, distribution_version: str) -> Path:
project = output.resolve()
try:
project.relative_to(BUILD_ROOT.resolve())
except ValueError as exc:
raise ValueError(f"Generated model project must be below {BUILD_ROOT}") from exc
if project == BUILD_ROOT.resolve():
raise ValueError("Generated model project cannot be the build root")
if project.exists():
shutil.rmtree(project)
project.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(
SOURCE_PROJECT,
project,
ignore=shutil.ignore_patterns(
"build",
"dist",
"*.egg-info",
"__pycache__",
"*.pyc",
"manifest.json",
"*.rxc",
"NOTICE-model-data.txt",
),
)
_replace_once(
project / "pyproject.toml",
r'^version = "0\.0\.0"$',
f'version = "{distribution_version}"',
)
_replace_once(
project / "radixor_models_standard" / "__init__.py",
r'^__version__ = "0\.0\.0"$',
f'__version__ = "{distribution_version}"',
)
return project
def _model_manifest(
model_id: str,
compile_model: Any,
package_root: Path,
reproducibility_directory: Path,
) -> dict[str, Any]:
project = REPOSITORY / "models" / model_id
source = project / "src" / "modelInput" / "stemmer.gz"
notice = project / "src" / "modelInput" / "NOTICE-model-data.txt"
version = (project / "model-version.txt").read_text(encoding="utf-8").strip()
if not source.is_file() or not notice.is_file() or not version:
raise FileNotFoundError(f"Incomplete canonical model project: {project}")
destination = package_root / "models" / f"{model_id}.rxc"
destination.parent.mkdir(parents=True, exist_ok=True)
compile_model(str(source), str(destination), language=model_id)
reproduction = reproducibility_directory / destination.name
compile_model(str(source), str(reproduction), language=model_id)
if destination.read_bytes() != reproduction.read_bytes():
raise ValueError(f"Non-deterministic compiled model output for {model_id}")
notice_destination = package_root / "notices" / model_id / notice.name
notice_destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(notice, notice_destination)
metadata = _gradle_metadata(project / "build.gradle")
if metadata["sourceLicense"] != "CC-BY-SA-3.0":
raise ValueError(f"Standard model {model_id} must be CC-BY-SA-3.0")
return {
"file": f"models/{model_id}.rxc",
"id": model_id,
"notice": f"notices/{model_id}/{notice.name}",
"provenance": {
"attribution": metadata["sourceAttribution"],
"dataset": metadata["sourceDataset"],
"license": metadata["sourceLicense"],
"license_uri": metadata["sourceLicenseUri"],
"repository": metadata["sourceRepository"],
"revision": metadata["sourceRevision"],
"revision_status": metadata["sourceRevisionStatus"],
"source_name": metadata["sourceName"],
"source_project": metadata["sourceProject"],
"source_version": metadata["sourceVersion"],
"transformations": metadata["transformationsSummary"],
"verification_date": metadata["sourceVerificationDate"],
},
"sha256": _sha256(destination),
"source": {
"path": f"models/{model_id}/src/modelInput/stemmer.gz",
"sha256": _sha256(source),
},
"version": version,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True, type=Path)
parser.add_argument("--distribution-version", required=True)
args = parser.parse_args()
if VERSION.fullmatch(args.distribution_version) is None:
raise SystemExit(
f"Invalid standard-model distribution version: {args.distribution_version!r}"
)
try:
from radixor import compile as compile_model
except ImportError as exc:
print(
"error: build the Radixor extension first (for example, "
"maturin develop --release) and rerun this script",
file=sys.stderr,
)
print(f"detail: {exc}", file=sys.stderr)
return 2
project = _stage_project(args.project, args.distribution_version)
package_root = project / "radixor_models_standard"
model_ids = _default_models()
expected_files = {f"{model_id}.rxc" for model_id in model_ids}
models_directory = package_root / "models"
models_directory.mkdir(parents=True, exist_ok=True)
for stale in models_directory.glob("*.rxc"):
if stale.name not in expected_files:
stale.unlink()
notices_directory = package_root / "notices"
notices_directory.mkdir(parents=True, exist_ok=True)
for stale in notices_directory.iterdir():
if stale.is_dir() and stale.name not in model_ids:
shutil.rmtree(stale)
reproducibility_root = BUILD_ROOT / "python" / "tmp"
reproducibility_root.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix="radixor-model-reproducibility-", dir=reproducibility_root
) as temporary:
models = [
_model_manifest(
model_id,
compile_model,
package_root,
Path(temporary),
)
for model_id in model_ids
]
manifest = {
"catalog_version": CATALOG_VERSION.read_text(encoding="utf-8").strip(),
"distribution_version": args.distribution_version,
"format": EXPECTED_FORMAT,
"models": models,
"schema_version": 1,
"topology": "models/model-projects.properties",
}
manifest_path = package_root / "manifest.json"
with manifest_path.open("w", encoding="utf-8", newline="\n") as stream:
stream.write(
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
)
print(f"generated {len(models)} standard compiled models in {package_root}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

132
python/scripts/prepare_release_tree.py Executable file
View File

@@ -0,0 +1,132 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Create an isolated, tag-versioned Python release source tree."""
from __future__ import annotations
import argparse
import re
import shutil
from pathlib import Path
REPOSITORY = Path(__file__).resolve().parents[2]
PYTHON_ROOT = REPOSITORY / "python"
BUILD_ROOT = REPOSITORY / "build"
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
text = path.read_text(encoding="utf-8")
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
if count != 1:
raise ValueError(f"Expected exactly one version field in {path}")
path.write_text(updated, encoding="utf-8", newline="\n")
def _prepare_output(path: Path) -> Path:
output = path.resolve()
try:
output.relative_to(BUILD_ROOT.resolve())
except ValueError as exc:
raise ValueError(f"Release staging output must be below {BUILD_ROOT}") from exc
if output == BUILD_ROOT.resolve():
raise ValueError("Release staging output cannot be the build root")
if output.exists():
shutil.rmtree(output)
output.parent.mkdir(parents=True, exist_ok=True)
return output
def _copy_native(output: Path, version: str) -> None:
shutil.copytree(
PYTHON_ROOT,
output,
ignore=shutil.ignore_patterns(
".gitignore",
".pytest_cache",
".ruff_cache",
".venv",
"__pycache__",
"*.pyc",
"_radixor*.dll",
"_radixor*.dylib",
"_radixor*.pyd",
"_radixor*.so",
"_native*.dll",
"_native*.dylib",
"_native*.pyd",
"_native*.so",
"benchmarks",
"dist",
"models",
"models-standard",
"target",
"tests",
),
)
_replace_once(
output / "pyproject.toml",
r'^version = "0\.0\.0"$',
f'version = "{version}"',
)
_replace_once(
output / "Cargo.toml",
r'^version = "0\.0\.0"$',
f'version = "{version}"',
)
lock = output / "Cargo.lock"
lock_text = lock.read_text(encoding="utf-8")
pattern = r'(\[\[package\]\]\nname = "radixor"\n)version = "0\.0\.0"'
lock_text, count = re.subn(pattern, rf'\g<1>version = "{version}"', lock_text)
if count != 1:
raise ValueError("Expected exactly one radixor package entry in Cargo.lock")
lock.write_text(lock_text, encoding="utf-8", newline="\n")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("distribution", choices=("native",))
parser.add_argument("version")
parser.add_argument("output", type=Path)
args = parser.parse_args()
if VERSION.fullmatch(args.version) is None:
raise SystemExit(f"Invalid stable release version: {args.version!r}")
output = _prepare_output(args.output)
_copy_native(output, args.version)
print(f"prepared {args.distribution} {args.version} in {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

173
python/scripts/update_simple_index.py Executable file
View File

@@ -0,0 +1,173 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Update Radixor's static PEP 503 index with verified GitHub Release assets."""
from __future__ import annotations
import argparse
import hashlib
import html
import re
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import quote
PACKAGES = ("radixor", "radixor-models-standard")
REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z")
SHA256 = re.compile(r"[0-9a-f]{64}\Z")
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
class _AnchorParser(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.anchors: dict[str, str] = {}
self._href: str | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag == "a":
self._href = dict(attrs).get("href")
def handle_data(self, data: str) -> None:
if self._href is not None and data.strip():
self.anchors[data.strip()] = self._href
def handle_endtag(self, tag: str) -> None:
if tag == "a":
self._href = None
def _existing_links(path: Path, expected_prefix: str) -> dict[str, str]:
if not path.exists():
return {}
parser = _AnchorParser()
parser.feed(path.read_text(encoding="utf-8"))
for filename, href in parser.anchors.items():
base, separator, digest = href.rpartition("#sha256=")
if (
not separator
or not base.startswith(expected_prefix)
or SHA256.fullmatch(digest) is None
or base.rsplit("/", 1)[-1] != quote(filename, safe="._-")
):
raise ValueError(f"Existing package-index link is unsafe: {href!r}")
return parser.anchors
def _render_root(root: Path) -> None:
links = "\n".join(
f' <a href="{html.escape(package)}/">{html.escape(package)}</a><br>'
for package in PACKAGES
)
(root / "index.html").write_text(
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
'content="1.0"><title>Radixor Python packages</title></head>\n'
f"<body>\n{links}\n</body></html>\n",
encoding="utf-8",
newline="\n",
)
def _render_project(path: Path, package: str, links: dict[str, str]) -> None:
anchors = "\n".join(
f' <a href="{html.escape(href, quote=True)}" '
f'data-requires-python="&gt;=3.9">{html.escape(filename)}</a><br>'
for filename, href in sorted(links.items())
)
path.write_text(
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
f'content="1.0"><title>Links for {html.escape(package)}</title></head>\n'
f"<body>\n{anchors}\n</body></html>\n",
encoding="utf-8",
newline="\n",
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--root", required=True, type=Path)
parser.add_argument("--repository", required=True)
parser.add_argument("--package", required=True, choices=PACKAGES)
parser.add_argument("--version", required=True)
parser.add_argument("--tag", required=True)
parser.add_argument("--artifacts", required=True, type=Path)
args = parser.parse_args()
if REPOSITORY.fullmatch(args.repository) is None:
raise SystemExit("Invalid GitHub repository identity")
if VERSION.fullmatch(args.version) is None:
raise SystemExit("Invalid stable release version")
expected_tag = (
f"python@{args.version}"
if args.package == "radixor"
else f"python-models-standard@{args.version}"
)
if args.tag != expected_tag:
raise SystemExit(f"Tag {args.tag!r} does not match {expected_tag!r}")
root = args.root.resolve()
if root.name != "simple" or root.parent.name != "python":
raise SystemExit("The package index must end in python/simple")
root.mkdir(parents=True, exist_ok=True)
project = root / args.package
project.mkdir(exist_ok=True)
page = project / "index.html"
prefix = f"https://github.com/{args.repository}/releases/download/"
links = _existing_links(page, prefix)
release_prefix = f"{prefix}{quote(args.tag, safe='@._-')}/"
artifacts = sorted(
path
for path in args.artifacts.iterdir()
if path.is_file()
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
)
if not artifacts:
raise SystemExit("No package artifacts were provided")
for artifact in artifacts:
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
href = f"{release_prefix}{quote(artifact.name, safe='._-')}#sha256={digest}"
previous = links.get(artifact.name)
if previous is not None and previous != href:
raise ValueError(
f"Refusing to replace existing index entry {artifact.name}"
)
links[artifact.name] = href
_render_root(root)
_render_project(page, args.package, links)
print(f"indexed {len(artifacts)} artifacts for {args.package} {args.version}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

598
python/scripts/verify_distributions.py Normal file
View File

@@ -0,0 +1,598 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Verify Radixor's native and standard-model release archives."""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import os
import stat
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from email.parser import BytesParser
from pathlib import Path, PurePosixPath
EXPECTED_DEFAULT_COUNT = 20
EXPECTED_DEPENDENCY = "radixor-models-standard>=1.0,<2.0"
REPOSITORY = Path(__file__).resolve().parents[2]
EXPECTED_NATIVE_LICENSE = REPOSITORY.joinpath("LICENSE").read_bytes()
EXPECTED_MODEL_IDS = {
"cs-cz-default",
"da-dk-default",
"de-de-default",
"es-es-default",
"fa-ir-default",
"fi-fi-default",
"fr-fr-default",
"he-il-default",
"hu-hu-default",
"it-it-default",
"nb-no-default",
"nl-nl-default",
"nn-no-default",
"pl-pl-unimorph",
"pt-pt-default",
"ru-ru-default",
"sv-se-default",
"uk-ua-default",
"us-uk-default",
"yi-default",
}
STANDARD_NAME = "radixor-models-standard"
STANDARD_VERSION = "0.0.0"
MAIN_NAME = "radixor"
MAIN_VERSION = "0.0.0"
def _one(directory: Path, pattern: str) -> Path:
matches = sorted(directory.glob(pattern))
if len(matches) != 1:
raise ValueError(
f"Expected exactly one {pattern} in {directory}, found {len(matches)}"
)
return matches[0]
def _assert_no_repository_temp_gzip() -> None:
offenders = sorted(
path.name for path in REPOSITORY.glob("tmp*.gz") if path.is_file()
)
if offenders:
raise ValueError(
"Repository root contains unmanaged Python temporary gzip files; "
f"remove them and fix the producing workflow: {offenders}"
)
def _wheel_metadata(archive: zipfile.ZipFile) -> object:
names = [
name for name in archive.namelist() if name.endswith(".dist-info/METADATA")
]
if len(names) != 1:
raise ValueError("Wheel must contain exactly one METADATA file")
return BytesParser().parsebytes(archive.read(names[0]))
def _normalized_member_name(name: str, archive_name: str) -> str:
if not name or "\\" in name or name.startswith("/"):
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
stripped = name[:-1] if name.endswith("/") else name
parts = stripped.split("/")
if not stripped or any(part in {"", ".", ".."} for part in parts):
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
return "/".join(parts)
def _validate_zip_members(archive: zipfile.ZipFile, archive_name: str) -> list[str]:
names: list[str] = []
seen: set[str] = set()
for info in archive.infolist():
name = _normalized_member_name(info.filename, archive_name)
if name in seen:
raise ValueError(f"{archive_name} contains duplicate member: {name}")
seen.add(name)
mode = (info.external_attr >> 16) & 0xFFFF
if stat.S_ISLNK(mode):
raise ValueError(f"{archive_name} contains symlink-like ZIP member: {name}")
names.append(name)
return names
def _validate_tar_members(archive: tarfile.TarFile, archive_name: str) -> list[str]:
names: list[str] = []
seen: set[str] = set()
for member in archive.getmembers():
name = _normalized_member_name(member.name, archive_name)
if name in seen:
raise ValueError(f"{archive_name} contains duplicate member: {name}")
seen.add(name)
if not (member.isfile() or member.isdir()):
raise ValueError(f"{archive_name} contains unsafe special member: {name}")
names.append(name)
return names
def _assert_distribution_metadata(
metadata: object, name: str, version: str, archive_name: str
) -> None:
if metadata.get("Name") != name or metadata.get("Version") != version:
raise ValueError(
f"{archive_name} has unexpected distribution identity: "
f"{metadata.get('Name')} {metadata.get('Version')}"
)
def _tar_file_bytes(archive: tarfile.TarFile, name: str) -> bytes:
member = archive.getmember(name)
stream = archive.extractfile(member)
if stream is None:
raise ValueError(f"Cannot read archive member: {name}")
return stream.read()
def _validate_standard_manifest(
manifest: object, expected_version: str = STANDARD_VERSION
) -> list[dict]:
if not isinstance(manifest, dict) or set(manifest) != {
"catalog_version",
"distribution_version",
"format",
"models",
"schema_version",
"topology",
}:
raise ValueError("Standard model manifest has an invalid top-level schema")
if (
manifest["schema_version"] != 1
or manifest["catalog_version"] != "2026.1"
or manifest["distribution_version"] != expected_version
or manifest["topology"] != "models/model-projects.properties"
or manifest["format"] != {"compression": "gzip", "magic": "EGTR", "version": 7}
or not isinstance(manifest["models"], list)
):
raise ValueError("Standard model manifest has incompatible catalog metadata")
models = manifest["models"]
if {
model.get("id") for model in models if isinstance(model, dict)
} != EXPECTED_MODEL_IDS:
raise ValueError(
"Standard model manifest does not match the default model topology"
)
if len(models) != len(EXPECTED_MODEL_IDS):
raise ValueError("Standard model manifest contains duplicate model entries")
provenance_keys = {
"attribution",
"dataset",
"license",
"license_uri",
"repository",
"revision",
"revision_status",
"source_name",
"source_project",
"source_version",
"transformations",
"verification_date",
}
for model in models:
if set(model) != {
"file",
"id",
"notice",
"provenance",
"sha256",
"source",
"version",
}:
raise ValueError("Standard model manifest contains an invalid model entry")
model_id = model["id"]
if (
model["file"] != f"models/{model_id}.rxc"
or model["notice"] != f"notices/{model_id}/NOTICE-model-data.txt"
or model["version"] != "1.0.0"
or not isinstance(model["sha256"], str)
or len(model["sha256"]) != 64
or set(model["provenance"]) != provenance_keys
or model["provenance"]["license"] != "CC-BY-SA-3.0"
or set(model["source"]) != {"path", "sha256"}
or model["source"]["path"] != f"models/{model_id}/src/modelInput/stemmer.gz"
):
raise ValueError(f"Standard model manifest entry is invalid: {model_id}")
if not all(isinstance(value, str) for value in model["provenance"].values()):
raise ValueError(f"Standard model provenance is invalid: {model_id}")
return models
def _assert_no_source_dictionaries(names: list[str], archive_name: str) -> None:
offenders = [
name
for name in names
if name.endswith("stemmer.gz") or "/radixor/models/" in name
]
if offenders:
raise ValueError(
f"{archive_name} contains forbidden source dictionaries: {offenders}"
)
def _assert_no_standard_model_payload(names: list[str], archive_name: str) -> None:
offenders = []
for name in names:
parts = PurePosixPath(name).parts
if (
name.endswith(".rxc")
or "models-standard" in parts
or any(part.startswith("radixor_models_standard") for part in parts)
):
offenders.append(name)
if offenders:
raise ValueError(
f"{archive_name} contains standard-model payload owned by "
f"radixor-models-standard: {offenders}"
)
def _assert_no_local_build_outputs(names: list[str], archive_name: str) -> None:
offenders = []
for name in names:
path = PurePosixPath(name)
parts = path.parts
is_benchmark_result = (
"benchmarks" in parts
and path.name.startswith("results")
and path.suffix in {".csv", ".json"}
)
if "dist" in parts or is_benchmark_result:
offenders.append(name)
if offenders:
raise ValueError(
f"{archive_name} contains local distribution or benchmark outputs: {offenders}"
)
def _assert_no_python_cache(
names: list[str], archive_name: str, *, allow_native_pyd: bool = False
) -> None:
offenders = []
for name in names:
path = PurePosixPath(name)
is_native_module = (
allow_native_pyd
and path.suffix == ".pyd"
and path.name.startswith("_radixor")
)
if (
"__pycache__" in path.parts
or path.suffix in {".pyc", ".pyo"}
or (path.suffix == ".pyd" and not is_native_module)
):
offenders.append(name)
if offenders:
raise ValueError(f"{archive_name} contains Python cache artifacts: {offenders}")
def _assert_main_dependency(metadata, archive_name: str) -> None:
requirements = metadata.get_all("Requires-Dist", [])
normalized = {requirement.replace(" ", "") for requirement in requirements}
if EXPECTED_DEPENDENCY not in normalized:
raise ValueError(
f"Missing compatible standard-model dependency in {archive_name}: "
f"{requirements}"
)
def _verify_main_wheel(path: Path, expected_version: str = MAIN_VERSION) -> None:
with zipfile.ZipFile(path) as archive:
names = _validate_zip_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_standard_model_payload(names, path.name)
_assert_no_python_cache(names, path.name, allow_native_pyd=True)
metadata = _wheel_metadata(archive)
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
if metadata.get("License-Expression") != "BSD-3-Clause":
raise ValueError(
f"{path.name} must declare License-Expression: BSD-3-Clause"
)
license_names = [
name for name in names if name.endswith(".dist-info/licenses/LICENSE")
]
if (
len(license_names) != 1
or archive.read(license_names[0]) != EXPECTED_NATIVE_LICENSE
):
raise ValueError(
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
)
_assert_main_dependency(metadata, path.name)
def _verify_main_sdist(path: Path, expected_version: str = MAIN_VERSION) -> None:
with tarfile.open(path, "r:gz") as archive:
names = _validate_tar_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_standard_model_payload(names, path.name)
_assert_no_local_build_outputs(names, path.name)
_assert_no_python_cache(names, path.name)
license_members = [
member
for member in archive.getmembers()
if PurePosixPath(member.name).name == "LICENSE" and member.isfile()
]
if len(license_members) != 1:
raise ValueError(
f"{path.name} must contain exactly one BSD-3-Clause LICENSE"
)
license_stream = archive.extractfile(license_members[0])
if license_stream is None or license_stream.read() != EXPECTED_NATIVE_LICENSE:
raise ValueError(
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
)
root = f"radixor-{expected_version}"
metadata = BytesParser().parsebytes(
_tar_file_bytes(archive, f"{root}/PKG-INFO")
)
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
_assert_main_dependency(metadata, path.name)
def _verify_standard_wheel(
path: Path, expected_version: str = STANDARD_VERSION
) -> None:
if not path.name.endswith("-py3-none-any.whl"):
raise ValueError(f"Standard model wheel is not pure py3-none-any: {path.name}")
with zipfile.ZipFile(path) as archive:
names = _validate_zip_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_python_cache(names, path.name)
metadata = _wheel_metadata(archive)
_assert_distribution_metadata(
metadata, STANDARD_NAME, expected_version, path.name
)
if metadata.get("License-Expression") != "CC-BY-SA-3.0":
raise ValueError("Standard model data must declare CC-BY-SA-3.0")
manifest = json.loads(archive.read("radixor_models_standard/manifest.json"))
models = _validate_standard_manifest(manifest, expected_version)
dist_info = next(
name.rsplit("/", 1)[0]
for name in names
if name.endswith(".dist-info/METADATA")
)
allowed = {
"radixor_models_standard/__init__.py",
"radixor_models_standard/manifest.json",
"radixor_models_standard/models/__init__.py",
"radixor_models_standard/notices/__init__.py",
f"{dist_info}/METADATA",
f"{dist_info}/WHEEL",
f"{dist_info}/RECORD",
f"{dist_info}/top_level.txt",
f"{dist_info}/licenses/LICENSE-MODEL-DATA.txt",
}
for model in models:
model_name = f"radixor_models_standard/{model['file']}"
notice_name = f"radixor_models_standard/{model['notice']}"
allowed.update({model_name, notice_name})
data = archive.read(model_name)
if hashlib.sha256(data).hexdigest() != model["sha256"]:
raise ValueError(f"Checksum mismatch for {model['id']}")
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
raise ValueError(f"Invalid v7 marker/version for {model['id']}")
if notice_name not in names:
raise ValueError(f"Missing model notice for {model['id']}")
unexpected = set(names) - allowed
missing = allowed - set(names)
if unexpected or missing:
raise ValueError(
f"Standard wheel allowlist mismatch; unexpected={sorted(unexpected)}, "
f"missing={sorted(missing)}"
)
for info in archive.infolist():
if info.filename in names and ((info.external_attr >> 16) & 0o111):
raise ValueError(
f"Standard wheel contains executable member: {info.filename}"
)
def _verify_standard_sdist(
path: Path, expected_version: str = STANDARD_VERSION
) -> None:
with tarfile.open(path, "r:gz") as archive:
names = _validate_tar_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_python_cache(names, path.name)
root = f"radixor_models_standard-{expected_version}"
metadata = BytesParser().parsebytes(
_tar_file_bytes(archive, f"{root}/PKG-INFO")
)
_assert_distribution_metadata(
metadata, STANDARD_NAME, expected_version, path.name
)
manifest_name = f"{root}/radixor_models_standard/manifest.json"
models = _validate_standard_manifest(
json.loads(_tar_file_bytes(archive, manifest_name)), expected_version
)
egg_info = f"{root}/radixor_models_standard.egg-info"
allowed_files = {
f"{root}/LICENSE-MODEL-DATA.txt",
f"{root}/MANIFEST.in",
f"{root}/PKG-INFO",
f"{root}/README.md",
f"{root}/pyproject.toml",
f"{root}/setup.cfg",
f"{root}/radixor_models_standard/__init__.py",
manifest_name,
f"{root}/radixor_models_standard/models/__init__.py",
f"{root}/radixor_models_standard/notices/__init__.py",
f"{egg_info}/PKG-INFO",
f"{egg_info}/SOURCES.txt",
f"{egg_info}/dependency_links.txt",
f"{egg_info}/top_level.txt",
}
allowed_dirs = {
root,
f"{root}/radixor_models_standard",
f"{root}/radixor_models_standard/models",
f"{root}/radixor_models_standard/notices",
egg_info,
}
for model in models:
model_name = f"{root}/radixor_models_standard/{model['file']}"
notice_name = f"{root}/radixor_models_standard/{model['notice']}"
notice_dir = notice_name.rsplit("/", 1)[0]
allowed_files.update({model_name, notice_name})
allowed_dirs.add(notice_dir)
data = _tar_file_bytes(archive, model_name)
if hashlib.sha256(data).hexdigest() != model["sha256"]:
raise ValueError(
f"Checksum mismatch for {model['id']} in standard sdist"
)
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
raise ValueError(
f"Invalid v7 marker/version for {model['id']} in standard sdist"
)
files = {member.name for member in archive.getmembers() if member.isfile()}
directories = {member.name for member in archive.getmembers() if member.isdir()}
if files != allowed_files or directories != allowed_dirs:
raise ValueError(
f"Standard sdist allowlist mismatch; unexpected files={sorted(files - allowed_files)}, "
f"missing files={sorted(allowed_files - files)}, "
f"unexpected dirs={sorted(directories - allowed_dirs)}, "
f"missing dirs={sorted(allowed_dirs - directories)}"
)
executable = [
member.name
for member in archive.getmembers()
if member.isfile() and member.mode & 0o111
]
if executable:
raise ValueError(
f"Standard sdist contains executable members: {executable}"
)
def _verify_fresh_install(main_wheel: Path, standard_wheel: Path) -> None:
managed_temp = REPOSITORY / "build" / "python" / "tmp"
managed_temp.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix="radixor-wheel-install-", dir=managed_temp
) as temporary:
environment = Path(temporary) / "venv"
subprocess.run([sys.executable, "-m", "venv", str(environment)], check=True)
scripts = "Scripts" if os.name == "nt" else "bin"
python = environment / scripts / ("python.exe" if os.name == "nt" else "python")
if "radixor_models_standard-0.0.0-" in standard_wheel.name:
subprocess.run(
[
str(python),
"-m",
"pip",
"install",
"--no-index",
"--no-deps",
str(standard_wheel),
str(main_wheel),
],
check=True,
)
else:
subprocess.run(
[
str(python),
"-m",
"pip",
"install",
"--no-index",
"--find-links",
str(main_wheel.parent),
"--find-links",
str(standard_wheel.parent),
"radixor",
],
check=True,
)
result = subprocess.run(
[
str(python),
"-c",
"from radixor import Stemmer; print(Stemmer('en').stem('running'))",
],
check=True,
capture_output=True,
text=True,
)
if result.stdout.strip() != "run":
raise ValueError(f"Unexpected installed model result: {result.stdout!r}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--main-wheel-dir", type=Path)
parser.add_argument("--main-sdist-dir", type=Path)
parser.add_argument("--standard-dir", type=Path)
parser.add_argument("--main-version", default=MAIN_VERSION)
parser.add_argument("--standard-version", default=STANDARD_VERSION)
parser.add_argument("--skip-install", action="store_true")
args = parser.parse_args()
_assert_no_repository_temp_gzip()
if (args.main_wheel_dir is None) != (args.main_sdist_dir is None):
parser.error("--main-wheel-dir and --main-sdist-dir must be used together")
if args.main_wheel_dir is None and args.standard_dir is None:
parser.error("at least one distribution directory must be provided")
main_wheel = None
standard_wheel = None
if args.main_wheel_dir is not None:
main_wheel = _one(args.main_wheel_dir, "radixor-*.whl")
main_sdist = _one(args.main_sdist_dir, "radixor-*.tar.gz")
_verify_main_wheel(main_wheel, args.main_version)
_verify_main_sdist(main_sdist, args.main_version)
if args.standard_dir is not None:
standard_wheel = _one(args.standard_dir, "radixor_models_standard-*.whl")
standard_sdist = _one(args.standard_dir, "radixor_models_standard-*.tar.gz")
_verify_standard_wheel(standard_wheel, args.standard_version)
_verify_standard_sdist(standard_sdist, args.standard_version)
if not args.skip_install and main_wheel is not None and standard_wheel is not None:
_verify_fresh_install(main_wheel, standard_wheel)
print("verified requested Radixor distributions")
return 0
if __name__ == "__main__":
raise SystemExit(main())

682
python/src/builder.rs Normal file
View File

@@ -0,0 +1,682 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// Port of the Radixor Java trie compilation pipeline
// (org.egothor.stemmer.FrequencyTrie.Builder + org.egothor.stemmer.trie.*):
// mutable trie build -> bottom-up reduction -> freeze to an immutable compiled trie.
//
// Faithful port notes:
// * Build semantics mirror StemmerPatchTrieLoader.load: for each dictionary
// entry we optionally insert the stem mapped to the NOOP patch "Na" (when
// store_original) and every variant != stem mapped to
// encode_patch(variant, stem, backward).
// * Keys are indexed per WordTraversalDirection: BACKWARD consumes characters
// right-to-left (logicalIndex = len-1-offset), FORWARD left-to-right.
// * Reduction hardcodes the production configuration verified from the Java
// source: ReductionMode = MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS,
// dominantWinnerMinPercent = 75, dominantWinnerOverSecondRatio = 3,
// contractUniformSubtrees = true (metadataForCompilation always applies
// ReductionSettings.withUniformSubtreeContraction).
// * All character/patch data is handled as UTF-16 code units (Java `char`),
// exactly as the runtime trie.rs expects.
#![allow(dead_code)]
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap};
use std::rc::Rc;
use std::sync::Arc;
use crate::dict::DictEntry;
use crate::encoder::encode_patch;
use crate::patch::PatchCommand;
use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata};
/// Canonical no-op patch command (PatchCommandEncoder.NOOP_PATCH = "Na").
const NOOP_PATCH: &str = "Na";
/// dominantWinnerMinPercent (ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT).
const DOMINANT_WINNER_MIN_PERCENT: i64 = 75;
/// dominantWinnerOverSecondRatio (ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO).
const DOMINANT_WINNER_OVER_SECOND_RATIO: i64 = 3;
// Ordered value-count map (Java LinkedHashMap<V, Integer> semantics)
/// Insertion-ordered map from a patch-command string to its accumulated local
/// frequency. Mirrors the `LinkedHashMap<V, Integer>` used for `valueCounts` on
/// mutable nodes and `localCounts` on reduced nodes.
#[derive(Clone, Default)]
struct OrderedCounts {
entries: Vec<(String, i32)>,
index: HashMap<String, usize>,
}
impl OrderedCounts {
fn new() -> Self {
OrderedCounts {
entries: Vec::new(),
index: HashMap::new(),
}
}
fn is_empty(&self) -> bool {
self.entries.is_empty()
}
fn len(&self) -> usize {
self.entries.len()
}
/// Adds `count` to `value`, preserving first-seen insertion order. This is
/// both the build-time `put` accumulation and the reduction-time
/// `mergeLocalCounts` aggregation.
fn add(&mut self, value: &str, count: i32) {
if let Some(&position) = self.index.get(value) {
self.entries[position].1 += count;
} else {
let position = self.entries.len();
self.index.insert(value.to_string(), position);
self.entries.push((value.to_string(), count));
}
}
}
// MutableNode (org.egothor.stemmer.trie.MutableNode)
/// Mutable build-time node: children indexed by transition character plus the
/// local terminal value counts stored exactly at this node.
struct MutableNode {
children: BTreeMap<u16, MutableNode>,
value_counts: OrderedCounts,
}
impl MutableNode {
fn new() -> Self {
MutableNode {
children: BTreeMap::new(),
value_counts: OrderedCounts::new(),
}
}
}
/// Stores a value at the node addressed by `key`, incrementing its local
/// frequency by one. Mirrors `FrequencyTrie.Builder.put`.
fn put(root: &mut MutableNode, key: &[u16], value: &str, backward: bool) {
let length = key.len();
let mut current = root;
for offset in 0..length {
// WordTraversalDirection.logicalIndex(length, offset).
let logical_index = if backward {
length - 1 - offset
} else {
offset
};
let edge = key[logical_index];
current = current
.children
.entry(edge)
.or_insert_with(MutableNode::new);
}
current.value_counts.add(value, 1);
}
// ReducedNode (org.egothor.stemmer.trie.ReducedNode)
/// Canonical reduced node used during subtree merging. Reduced nodes are shared:
/// there is exactly one instance per reduction signature, referenced through
/// `Rc` so that identical subtrees share a single instance (and therefore a
/// single frozen `Arc<CompiledNode>`).
struct ReducedNode {
/// Canonical reduction signature (see `compute_signature`).
signature: String,
/// Aggregated local value counts.
local_counts: OrderedCounts,
/// Canonical children by edge, naturally sorted ascending by the BTreeMap.
children: BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
/// Whether this node is a contracted accepting leaf.
accepts: bool,
}
impl ReducedNode {
/// Merges additional local counts into this canonical node.
fn merge_local_counts(&mut self, additional: &OrderedCounts) {
for (value, count) in &additional.entries {
self.local_counts.add(value, *count);
}
}
/// Merges child references into this canonical node. For nodes with the same
/// reduction signature the child edge sets and child signatures are
/// compatible, so this only verifies canonical identity and stores it.
fn merge_children(&mut self, additional: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>) {
for (edge, child) in additional {
match self.children.get(edge) {
Some(existing) => {
if !Rc::ptr_eq(existing, child) {
panic!("Incompatible canonical child encountered during reduction.");
}
}
None => {
self.children.insert(*edge, Rc::clone(child));
}
}
}
}
}
// LocalValueSummary (org.egothor.stemmer.trie.LocalValueSummary)
/// Deterministic local terminal value summary of a node.
struct LocalValueSummary {
/// Locally stored values ordered by descending frequency, then shorter text,
/// then lexicographic (UTF-16) text, then first-seen insertion order.
ordered_values: Vec<String>,
/// Frequencies aligned with `ordered_values` (needed for v7 serialization).
ordered_counts: Vec<i32>,
total_count: i64,
dominant_value: Option<String>,
dominant_count: i64,
second_count: i64,
}
impl LocalValueSummary {
/// Builds a summary from local counts, applying the exact Java ordering.
fn of(counts: &OrderedCounts) -> Self {
struct Sortable {
value: String,
count: i32,
// Java String.length() and String.compareTo operate on UTF-16 code
// units, so text ordering must compare the u16 sequence, never UTF-8.
text16: Vec<u16>,
insertion_order: usize,
}
let mut entries: Vec<Sortable> = counts
.entries
.iter()
.enumerate()
.map(|(insertion_order, (value, count))| Sortable {
value: value.clone(),
count: *count,
text16: value.encode_utf16().collect(),
insertion_order,
})
.collect();
entries.sort_by(|left, right| {
// 1. descending frequency
right
.count
.cmp(&left.count)
// 2. shorter text wins
.then_with(|| left.text16.len().cmp(&right.text16.len()))
// 3. lexicographically lower text (UTF-16 code units) wins
.then_with(|| left.text16.cmp(&right.text16))
// 4. stable first-seen insertion order
.then_with(|| left.insertion_order.cmp(&right.insertion_order))
});
let ordered_values: Vec<String> = entries.iter().map(|entry| entry.value.clone()).collect();
let ordered_counts: Vec<i32> = entries.iter().map(|entry| entry.count).collect();
let total_count: i64 = entries.iter().map(|entry| entry.count as i64).sum();
let dominant_value = entries.first().map(|entry| entry.value.clone());
let dominant_count = entries.first().map(|entry| entry.count as i64).unwrap_or(0);
let second_count = entries.get(1).map(|entry| entry.count as i64).unwrap_or(0);
LocalValueSummary {
ordered_values,
ordered_counts,
total_count,
dominant_value,
dominant_count,
second_count,
}
}
/// Whether the dominant value satisfies both configured dominance
/// constraints (percent AND ratio), matching
/// `LocalValueSummary.hasQualifiedDominantWinner`.
fn has_qualified_dominant_winner(&self) -> bool {
if self.dominant_value.is_none() {
return false;
}
let percent_satisfied =
self.dominant_count * 100 >= self.total_count * DOMINANT_WINNER_MIN_PERCENT;
let ratio_satisfied = if self.second_count == 0 {
true
} else {
self.dominant_count >= self.second_count * DOMINANT_WINNER_OVER_SECOND_RATIO
};
percent_satisfied && ratio_satisfied
}
}
// ReductionSignature (org.egothor.stemmer.trie.ReductionSignature and friends)
/// Appends `text` to `buffer` using a length-prefixed, collision-free encoding
/// so arbitrary UTF-16 patch strings can be embedded without ambiguity.
fn push_len_prefixed(buffer: &mut String, text: &str) {
buffer.push_str(&text.len().to_string());
buffer.push('#');
buffer.push_str(text);
}
/// Produces the canonical reduction signature of a subtree as an unambiguous
/// hashable string. Two subtrees receive equal signatures exactly when the Java
/// `ReductionSignature.equals` would consider them equal:
///
/// * local descriptor — for DOMINANT mode this is the dominant descriptor
/// (only the dominant value) when the summary has a qualified dominant
/// winner, otherwise the ranked descriptor (the full ordered value list),
/// * whether the node accepts remaining input,
/// * the sorted list of (edge label, child signature) pairs.
fn compute_signature(
summary: &LocalValueSummary,
children: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
accepts: bool,
) -> String {
let mut signature = String::new();
// Local descriptor. 'D' and 'R' markers keep a DominantLocalDescriptor
// distinct from a RankedLocalDescriptor holding the same single value,
// exactly as the Java class-based equality does.
if summary.has_qualified_dominant_winner() {
signature.push('D');
push_len_prefixed(&mut signature, summary.dominant_value.as_ref().unwrap());
} else {
signature.push('R');
signature.push_str(&summary.ordered_values.len().to_string());
signature.push(';');
for value in &summary.ordered_values {
push_len_prefixed(&mut signature, value);
}
}
// acceptsRemainingInput.
signature.push(if accepts { 'A' } else { 'a' });
// Child descriptors in sorted edge order (BTreeMap iterates ascending).
signature.push_str(&children.len().to_string());
signature.push(';');
for (label, child) in children {
signature.push_str(&label.to_string());
signature.push(':');
push_len_prefixed(&mut signature, &child.borrow().signature);
}
signature
}
/// Returns aggregated single-value local counts when the supplied internal
/// subtree can be contracted into an accepting leaf, otherwise `None`.
///
/// Contraction applies (matching `FrequencyTrie.Builder.contractUniformSubtree`)
/// when the node has at least one child, every child is a single-value leaf with
/// no further children, and all those child values plus the local value (if any)
/// are the same single value. The contracted count is always 1.
fn contract_uniform_subtree(
local_counts: &OrderedCounts,
children: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
) -> Option<OrderedCounts> {
if children.is_empty() {
return None;
}
let mut uniform_value: Option<String> = None;
let mut value_seen = false;
if !local_counts.is_empty() {
if local_counts.len() != 1 {
return None;
}
uniform_value = Some(local_counts.entries[0].0.clone());
value_seen = true;
}
for child in children.values() {
let child_ref = child.borrow();
let is_single_value_leaf =
child_ref.children.is_empty() && child_ref.local_counts.len() == 1;
if !is_single_value_leaf {
return None;
}
let child_value = child_ref.local_counts.entries[0].0.clone();
if value_seen && uniform_value.as_deref() != Some(child_value.as_str()) {
return None;
}
uniform_value = Some(child_value);
value_seen = true;
}
if !value_seen {
return None;
}
let mut contracted = OrderedCounts::new();
contracted.add(uniform_value.as_ref().unwrap(), 1);
Some(contracted)
}
/// Reduces a mutable node to a canonical reduced node (bottom-up).
///
/// The order of operations mirrors the Java `reduce`:
/// 1. reduce every child first,
/// 2. try `contractUniformSubtree` (always enabled here),
/// 3. compute the local summary and reduction signature,
/// 4. deduplicate through the context map, merging counts and children into
/// an existing canonical node when the signature already exists.
fn reduce(
node: &MutableNode,
context: &mut HashMap<String, Rc<RefCell<ReducedNode>>>,
) -> Rc<RefCell<ReducedNode>> {
let mut reduced_children: BTreeMap<u16, Rc<RefCell<ReducedNode>>> = BTreeMap::new();
for (edge, child) in node.children.iter() {
let reduced_child = reduce(child, context);
reduced_children.insert(*edge, reduced_child);
}
let mut local_counts = node.value_counts.clone();
let mut accepts_remaining_input = false;
// contractUniformSubtrees is always true for the production configuration.
if let Some(contracted) = contract_uniform_subtree(&local_counts, &reduced_children) {
local_counts = contracted;
reduced_children = BTreeMap::new();
accepts_remaining_input = true;
}
let summary = LocalValueSummary::of(&local_counts);
let signature = compute_signature(&summary, &reduced_children, accepts_remaining_input);
if let Some(canonical) = context.get(&signature).cloned() {
{
let mut canonical_mut = canonical.borrow_mut();
canonical_mut.merge_local_counts(&local_counts);
canonical_mut.merge_children(&reduced_children);
}
return canonical;
}
let canonical = Rc::new(RefCell::new(ReducedNode {
signature: signature.clone(),
local_counts,
children: reduced_children,
accepts: accepts_remaining_input,
}));
context.insert(signature, Rc::clone(&canonical));
canonical
}
// Freeze (FrequencyTrie.Builder.freeze -> flat CSR arrays)
/// Maximum contiguous child-label span for which a node uses a dense
/// direct-index table instead of binary search (mirrors the Java
/// CompiledNode `maxExpandedIndex` fanout strategy).
pub(crate) const MAX_DENSE_SPAN: usize = 512;
/// Frozen arrays of the compiled trie in CSR layout (see trie.rs).
pub(crate) struct FrozenTrie {
pub(crate) edge_start: Vec<u32>,
pub(crate) edge_labels: Vec<u16>,
pub(crate) edge_targets: Vec<u32>,
pub(crate) accepts: Vec<bool>,
pub(crate) value_start: Vec<u32>,
pub(crate) values: Vec<Arc<PatchCommand>>,
/// Patch strings parallel to `values` (needed only for serialization).
pub(crate) value_strings: Vec<String>,
/// Frequencies parallel to `values` (needed only for v7 serialization).
pub(crate) value_counts: Vec<i32>,
pub(crate) dense_start: Vec<u32>,
pub(crate) dense_base: Vec<u16>,
pub(crate) dense_targets: Vec<u32>,
}
/// Per-node build record collected during interning, in node-id order.
#[derive(Default)]
struct NodeBuild {
edges: Vec<u16>,
targets: Vec<u32>,
accepts: bool,
values: Vec<Arc<PatchCommand>>,
value_strings: Vec<String>,
value_counts: Vec<i32>,
}
/// Assigns a stable node id to each distinct canonical reduced node and records
/// its edges (ascending), child ids, and best-first values.
///
/// Shared canonical reduced nodes (identical `Rc` allocations) are interned once
/// — the analogue of the Java `IdentityHashMap<ReducedNode, CompiledNode>` cache
/// — so structural sharing from reduction is preserved as shared node ids. Equal
/// patch strings are compiled once and shared through `patch_cache`.
fn intern(
node: &Rc<RefCell<ReducedNode>>,
index_of: &mut HashMap<usize, u32>,
nodes: &mut Vec<NodeBuild>,
patch_cache: &mut HashMap<String, Arc<PatchCommand>>,
backward: bool,
) -> u32 {
let identity = Rc::as_ptr(node) as usize;
if let Some(&existing) = index_of.get(&identity) {
return existing;
}
let id = nodes.len() as u32;
index_of.insert(identity, id);
nodes.push(NodeBuild::default()); // reserve this id's slot before recursing
let node_ref = node.borrow();
let summary = LocalValueSummary::of(&node_ref.local_counts);
// BTreeMap iterates ascending by edge label, so edges stay sorted.
let mut edges: Vec<u16> = Vec::with_capacity(node_ref.children.len());
let mut targets: Vec<u32> = Vec::with_capacity(node_ref.children.len());
for (edge, child) in node_ref.children.iter() {
edges.push(*edge);
targets.push(intern(child, index_of, nodes, patch_cache, backward));
}
let values: Vec<Arc<PatchCommand>> = summary
.ordered_values
.iter()
.map(|patch| {
Arc::clone(
patch_cache
.entry(patch.clone())
.or_insert_with(|| Arc::new(PatchCommand::parse(patch, backward))),
)
})
.collect();
nodes[id as usize] = NodeBuild {
edges,
targets,
accepts: node_ref.accepts,
values,
value_strings: summary.ordered_values.clone(),
value_counts: summary.ordered_counts.clone(),
};
id
}
/// Freezes the reduced graph rooted at `root` (node id 0) into flat CSR arrays.
fn freeze(root: &Rc<RefCell<ReducedNode>>, backward: bool) -> FrozenTrie {
let mut index_of: HashMap<usize, u32> = HashMap::new();
let mut nodes: Vec<NodeBuild> = Vec::new();
let mut patch_cache: HashMap<String, Arc<PatchCommand>> = HashMap::new();
intern(root, &mut index_of, &mut nodes, &mut patch_cache, backward);
let node_count = nodes.len();
let mut edge_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut edge_labels: Vec<u16> = Vec::new();
let mut edge_targets: Vec<u32> = Vec::new();
let mut accepts: Vec<bool> = Vec::with_capacity(node_count);
let mut value_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut values: Vec<Arc<PatchCommand>> = Vec::new();
let mut value_strings: Vec<String> = Vec::new();
let mut value_counts: Vec<i32> = Vec::new();
let mut dense_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut dense_base: Vec<u16> = Vec::with_capacity(node_count);
let mut dense_targets: Vec<u32> = Vec::new();
edge_start.push(0);
value_start.push(0);
dense_start.push(0);
for nb in &nodes {
edge_labels.extend_from_slice(&nb.edges);
edge_targets.extend_from_slice(&nb.targets);
edge_start.push(edge_labels.len() as u32);
accepts.push(nb.accepts);
for v in &nb.values {
values.push(Arc::clone(v));
}
for v in &nb.value_strings {
value_strings.push(v.clone());
}
value_counts.extend_from_slice(&nb.value_counts);
value_start.push(values.len() as u32);
// Decide dense vs sparse child lookup by fanout/span.
let count = nb.edges.len();
let mut dense = false;
if count >= 2 {
let first = nb.edges[0] as usize;
let last = nb.edges[count - 1] as usize; // edges are ascending
let span = last - first + 1;
if span <= MAX_DENSE_SPAN {
let base = nb.edges[0];
let seg = dense_targets.len();
dense_targets.resize(seg + span, 0);
for (k, &label) in nb.edges.iter().enumerate() {
dense_targets[seg + (label - base) as usize] = nb.targets[k] + 1;
}
dense_base.push(base);
dense_start.push(dense_targets.len() as u32);
dense = true;
}
}
if !dense {
dense_base.push(0);
dense_start.push(dense_targets.len() as u32); // span 0 => sparse
}
}
FrozenTrie {
edge_start,
edge_labels,
edge_targets,
accepts,
value_start,
values,
value_strings,
value_counts,
dense_start,
dense_base,
dense_targets,
}
}
pub(crate) fn metadata_for(backward: bool, lowercase: bool) -> TrieMetadata {
TrieMetadata {
traversal: if backward {
TraversalDirection::Backward
} else {
TraversalDirection::Forward
},
case_mode: if lowercase {
CaseMode::LowercaseWithLocaleRoot
} else {
CaseMode::AsIs
},
diacritic_mode: DiacriticMode::AsIs,
}
}
/// Build the reduced+frozen trie arrays from dictionary entries (shared by the
/// in-memory builder and the compiler).
pub(crate) fn build_frozen(
entries: &[DictEntry],
backward: bool,
store_original: bool,
) -> FrozenTrie {
let mut root = MutableNode::new();
for entry in entries {
let stem16: Vec<u16> = entry.stem.encode_utf16().collect();
if store_original {
put(&mut root, &stem16, NOOP_PATCH, backward);
}
for variant in &entry.variants {
if variant != &entry.stem {
let variant16: Vec<u16> = variant.encode_utf16().collect();
let patch = encode_patch(&variant16, &stem16, backward);
put(&mut root, &variant16, &patch, backward);
}
}
}
let mut context: HashMap<String, Rc<RefCell<ReducedNode>>> = HashMap::new();
let reduced_root = reduce(&root, &mut context);
freeze(&reduced_root, backward)
}
fn frozen_into_trie(frozen: FrozenTrie, metadata: TrieMetadata) -> FrequencyTrie {
FrequencyTrie::new(
frozen.edge_start,
frozen.edge_labels,
frozen.edge_targets,
frozen.accepts,
frozen.value_start,
frozen.values,
frozen.dense_start,
frozen.dense_base,
frozen.dense_targets,
metadata,
)
}
// Public entry point
/// Compiles dictionary entries into a read-only patch-command trie, faithfully
/// reproducing the Java `StemmerPatchTrieLoader.load` build followed by
/// `FrequencyTrie.Builder.build` (reduce + freeze).
///
/// * `backward` — `true` selects BACKWARD traversal (all languages except the
/// right-to-left fa/he/yi), `false` selects FORWARD.
/// * `store_original` — when `true`, each stem is inserted mapped to the NOOP
/// patch `"Na"` so the stem itself is recognised.
pub fn build_trie_from_dict(
entries: &[DictEntry],
backward: bool,
store_original: bool,
lowercase: bool,
) -> FrequencyTrie {
let frozen = build_frozen(entries, backward, store_original);
frozen_into_trie(frozen, metadata_for(backward, lowercase))
}

122
python/src/dict.rs Normal file
View File

@@ -0,0 +1,122 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// Port of StemmerDictionaryParser (Java) — line-oriented, tab-separated dictionary.
//
// Layout: first column = canonical stem, following tab-separated columns = variants.
// Remarks: the earliest occurrence of `#` or `//` terminates the logical line.
// Case: LOWERCASE_WITH_LOCALE_ROOT lowercases the line (locale-independent here).
// Items containing any whitespace character are ignored (Java: Character.isWhitespace).
use flate2::read::GzDecoder;
use std::io::{self, Read};
/// One parsed dictionary entry: a canonical stem and its accepted variants,
/// in encounter order.
pub struct DictEntry {
pub stem: String,
pub variants: Vec<String>,
}
/// Decompress gzipped UTF-8 dictionary bytes and parse them into entries.
/// `lowercase` mirrors CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT.
#[allow(dead_code)] // public helper; the runtime path decompresses then parse_text
pub fn parse_gz_dict(compressed: &[u8], lowercase: bool) -> io::Result<Vec<DictEntry>> {
let mut decoder = GzDecoder::new(compressed);
let mut text = String::new();
decoder.read_to_string(&mut text)?;
Ok(parse_text(&text, lowercase))
}
/// Parse an already-decompressed dictionary text.
pub fn parse_text(text: &str, lowercase: bool) -> Vec<DictEntry> {
let mut entries = Vec::new();
for raw_line in text.lines() {
// stripRemark(line).trim(), then lowercase.
let stripped = strip_remark(raw_line).trim();
if stripped.is_empty() {
continue;
}
let normalized: String = if lowercase {
stripped.to_lowercase()
} else {
stripped.to_string()
};
if normalized.is_empty() {
continue;
}
// split on '\t' keeping trailing empties (Java split("\t", -1)).
let mut columns = normalized.split('\t');
let stem = match columns.next() {
Some(c) => c.trim(),
None => continue,
};
if stem.is_empty() || contains_whitespace(stem) {
continue;
}
let mut variants = Vec::new();
for col in columns {
let variant = col.trim();
if variant.is_empty() || contains_whitespace(variant) {
continue;
}
variants.push(variant.to_string());
}
entries.push(DictEntry {
stem: stem.to_string(),
variants,
});
}
entries
}
/// Removes a trailing remark: the earliest of `#` or `//` terminates the line.
fn strip_remark(line: &str) -> &str {
let hash = line.find('#');
let slash = line.find("//");
let remark = match (hash, slash) {
(None, None) => return line,
(Some(h), None) => h,
(None, Some(s)) => s,
(Some(h), Some(s)) => h.min(s),
};
&line[..remark]
}
/// Matches Java Character.isWhitespace closely enough for dictionary items.
#[inline]
fn contains_whitespace(item: &str) -> bool {
item.chars().any(|c| c.is_whitespace())
}

349
python/src/encoder.rs Normal file
View File

@@ -0,0 +1,349 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// Port of PatchCommandEncoder (Java) — DP-based minimum-cost edit script.
// Costs: insert=1, delete=1, replace=1, match=0, mismatch_penalty=100.
// Produces compact opcode strings: D(elete), I(nsert), R(eplace), -(skip), N(oop).
// Count argument: 'a' + count - 1 (i.e., COUNT_SENTINEL = 'a' - 1 = 96).
const MISMATCH_PENALTY: i32 = 100;
const COUNT_SENTINEL: u16 = b'a' as u16 - 1; // 96 = 0x60
#[derive(Clone, Copy, PartialEq)]
enum Trace {
Delete,
Insert,
Replace,
Match,
}
/// Encode the patch command that transforms `source` (UTF-16 slice) into `target`.
/// Returns "Na" when source == target.
pub fn encode_patch(source: &[u16], target: &[u16], backward: bool) -> String {
if source == target {
return "Na".to_string();
}
if backward {
encode_backward(source, target)
} else {
encode_forward(source, target)
}
}
// Backward traversal encoding.
fn encode_backward(source: &[u16], target: &[u16]) -> String {
let src_len = source.len();
let tgt_len = target.len();
let cols = tgt_len + 1;
let mut cost = vec![0i32; (src_len + 1) * cols];
let mut trace = vec![Trace::Match; (src_len + 1) * cols];
let idx = |r: usize, c: usize| r * cols + c;
// Boundary conditions (Egothor backward: rows=source, cols=target)
for i in 1..=src_len {
cost[idx(i, 0)] = i as i32;
trace[idx(i, 0)] = Trace::Delete;
}
for j in 1..=tgt_len {
cost[idx(0, j)] = j as i32;
trace[idx(0, j)] = Trace::Insert;
}
// Fill left-to-right, top-to-bottom (sourceIndex 1..=srcLen, targetIndex 1..=tgtLen)
for si in 1..=src_len {
let src_ch = source[si - 1]; // sourceCharacters[sourceIndex + sourceCharacterOffset=-1]
for ti in 1..=tgt_len {
let tgt_ch = target[ti - 1];
// sourceNeighbor = sourceIndex - 1, targetNeighbor = targetIndex - 1
let del = cost[idx(si - 1, ti)] + 1; // DELETE from [si-1][ti]
let ins = cost[idx(si, ti - 1)] + 1; // INSERT from [si][ti-1]
let diag = cost[idx(si - 1, ti - 1)];
let rep = diag + 1;
let mat = diag
+ if src_ch == tgt_ch {
0
} else {
MISMATCH_PENALTY
};
// Priority: MATCH (baseline), then DELETE (<=), INSERT (<), REPLACE (<)
let mut best = mat;
let mut bt = Trace::Match;
if del <= best {
best = del;
bt = Trace::Delete;
}
if ins < best {
best = ins;
bt = Trace::Insert;
}
if rep < best {
bt = Trace::Replace;
}
let _ = best;
cost[idx(si, ti)] = if bt == Trace::Replace {
rep
} else if bt == Trace::Insert {
ins
} else if bt == Trace::Delete {
del
} else {
mat
};
trace[idx(si, ti)] = bt;
}
}
build_patch_backward(&trace, target, cols, src_len, tgt_len)
}
fn build_patch_backward(
trace: &[Trace],
target: &[u16],
cols: usize,
src_len: usize,
tgt_len: usize,
) -> String {
let idx = |r: usize, c: usize| r * cols + c;
let mut patch = String::new();
let mut pending_deletes: u16 = COUNT_SENTINEL;
let mut pending_skips: u16 = COUNT_SENTINEL;
let mut si = src_len;
let mut ti = tgt_len;
while si != 0 || ti != 0 {
match trace[idx(si, ti)] {
Trace::Delete => {
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
pending_deletes = pending_deletes.wrapping_add(1);
si -= 1;
}
Trace::Insert => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
ti -= 1;
append_instruction(&mut patch, 'I', target[ti]);
}
Trace::Replace => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
ti -= 1;
si -= 1;
append_instruction(&mut patch, 'R', target[ti]);
}
Trace::Match => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
pending_skips = pending_skips.wrapping_add(1);
si -= 1;
ti -= 1;
}
}
}
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
}
patch
}
// Forward traversal encoding.
fn encode_forward(source: &[u16], target: &[u16]) -> String {
let src_len = source.len();
let tgt_len = target.len();
let cols = tgt_len + 1;
let mut cost = vec![0i32; (src_len + 1) * cols];
let mut trace = vec![Trace::Match; (src_len + 1) * cols];
let idx = |r: usize, c: usize| r * cols + c;
// Boundary conditions (fill from bottom-right corner)
// cost[srcLen][tgtLen] = 0, trace = MATCH
for si in (0..src_len).rev() {
cost[idx(si, tgt_len)] = cost[idx(si + 1, tgt_len)] + 1;
trace[idx(si, tgt_len)] = Trace::Delete;
}
for ti in (0..tgt_len).rev() {
cost[idx(src_len, ti)] = cost[idx(src_len, ti + 1)] + 1;
trace[idx(src_len, ti)] = Trace::Insert;
}
// Fill right-to-left, bottom-to-top
for si in (0..src_len).rev() {
let src_ch = source[si]; // sourceCharacters[sourceIndex + sourceCharacterOffset=0]
for ti in (0..tgt_len).rev() {
let tgt_ch = target[ti];
// sourceNeighbor = sourceIndex + 1, targetNeighbor = targetIndex + 1
let del = cost[idx(si + 1, ti)] + 1;
let ins = cost[idx(si, ti + 1)] + 1;
let diag = cost[idx(si + 1, ti + 1)];
let rep = diag + 1;
let mat = diag
+ if src_ch == tgt_ch {
0
} else {
MISMATCH_PENALTY
};
let mut best = mat;
let mut bt = Trace::Match;
if del <= best {
best = del;
bt = Trace::Delete;
}
if ins < best {
best = ins;
bt = Trace::Insert;
}
if rep < best {
bt = Trace::Replace;
}
let _ = best;
cost[idx(si, ti)] = if bt == Trace::Replace {
rep
} else if bt == Trace::Insert {
ins
} else if bt == Trace::Delete {
del
} else {
mat
};
trace[idx(si, ti)] = bt;
}
}
build_patch_forward(&trace, target, cols, src_len, tgt_len)
}
fn build_patch_forward(
trace: &[Trace],
target: &[u16],
cols: usize,
src_len: usize,
tgt_len: usize,
) -> String {
let idx = |r: usize, c: usize| r * cols + c;
let mut patch = String::new();
let mut pending_deletes: u16 = COUNT_SENTINEL;
let mut pending_skips: u16 = COUNT_SENTINEL;
let mut si = 0usize;
let mut ti = 0usize;
while si != src_len || ti != tgt_len {
match trace[idx(si, ti)] {
Trace::Delete => {
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
pending_deletes = pending_deletes.wrapping_add(1);
si += 1;
}
Trace::Insert => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
append_instruction(&mut patch, 'I', target[ti]);
ti += 1;
}
Trace::Replace => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
if pending_skips != COUNT_SENTINEL {
append_instruction(&mut patch, '-', pending_skips);
pending_skips = COUNT_SENTINEL;
}
append_instruction(&mut patch, 'R', target[ti]);
si += 1;
ti += 1;
}
Trace::Match => {
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
pending_deletes = COUNT_SENTINEL;
}
pending_skips = pending_skips.wrapping_add(1);
si += 1;
ti += 1;
}
}
}
if pending_deletes != COUNT_SENTINEL {
append_instruction(&mut patch, 'D', pending_deletes);
}
patch
}
// Instruction encoding helpers.
#[inline]
fn append_instruction(patch: &mut String, opcode: char, argument: u16) {
patch.push(opcode);
patch.push(char::from_u32(argument as u32).unwrap_or('\u{FFFD}'));
}

346
python/src/lib.rs Normal file
View File

@@ -0,0 +1,346 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
mod builder;
mod dict;
mod encoder;
mod patch;
mod serial;
mod trie;
use flate2::read::GzDecoder;
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedStr;
use pyo3::types::{PyList, PyString};
use std::collections::HashMap;
use std::fs;
use std::io::Read;
use std::sync::{Arc, Mutex};
use trie::FrequencyTrie;
/// Decompress a gzip byte image, or return the bytes unchanged when they are
/// not gzip-framed (so plain-text dictionaries also work).
fn decompress_or_raw(bytes: &[u8]) -> Vec<u8> {
if bytes.len() >= 2 && bytes[0] == 0x1F && bytes[1] == 0x8B {
let mut out = Vec::new();
if GzDecoder::new(bytes).read_to_end(&mut out).is_ok() {
return out;
}
}
bytes.to_vec()
}
/// Decode UTF-16 code units into a reused UTF-8 buffer (lossy on unpaired
/// surrogates, which never occur in valid patch output).
#[inline]
fn decode_utf16_into(units: &[u16], out: &mut String) {
out.clear();
for r in char::decode_utf16(units.iter().copied()) {
out.push(r.unwrap_or('\u{FFFD}'));
}
}
/// Runtime stemmer core: compiles a gzipped textual dictionary into a
/// patch-command trie (in Rust) and stems words against it.
#[pyclass(module = "radixor._radixor")]
struct StemmerCore {
trie: Arc<FrequencyTrie>,
// Optional result cache (like PyStemmer's): maps an input word to the
// already-built Python result object (a str, or None). A hit is a refcount
// bump — no re-stemming and no new string. Disabled when `cache_cap == 0`.
cache: Option<Mutex<HashMap<String, Py<PyAny>>>>,
cache_cap: usize,
}
impl StemmerCore {
fn stem_cached(
&self,
py: Python<'_>,
word: &str,
key_buf: &mut Vec<u16>,
u16_buf: &mut Vec<u16>,
u8_buf: &mut String,
) -> Py<PyAny> {
let may_insert = if let Some(cache) = &self.cache {
let map = cache.lock().unwrap();
if let Some(obj) = map.get(word) {
return obj.clone_ref(py);
}
map.len() < self.cache_cap
} else {
false
};
let computed: Py<PyAny> = match self.trie.stem_len_into(word, key_buf, u16_buf) {
Some(_) => {
decode_utf16_into(u16_buf, u8_buf);
PyString::new_bound(py, u8_buf).into_any().unbind()
}
None => py.None(),
};
// A full insertion-only cache cannot become writable again, so avoid
// a second lock and hash probe for later distinct words.
if may_insert {
let cache = self.cache.as_ref().expect("enabled cache");
let mut map = cache.lock().unwrap();
// Another thread may have populated this word while this thread
// was stemming it. Return the shared cached object when it did.
if let Some(obj) = map.get(word) {
return obj.clone_ref(py);
}
if map.len() < self.cache_cap {
map.insert(word.to_owned(), computed.clone_ref(py));
}
}
computed
}
fn stem_batch_impl<'py>(
&self,
py: Python<'py>,
words: &[PyBackedStr],
fallback_to_original: bool,
) -> PyResult<Bound<'py, PyList>> {
let mut key_buf: Vec<u16> = Vec::new();
let mut u16_buf: Vec<u16> = Vec::new();
let mut u8_buf = String::new();
let list = PyList::empty_bound(py);
// Misses remain cached as None so calls through the compatibility API
// cannot change the existing stem/stem_batch missing-value contract.
for w in words {
let key: &str = w;
let obj = self.stem_cached(py, key, &mut key_buf, &mut u16_buf, &mut u8_buf);
if fallback_to_original && obj.bind(py).is_none() {
list.append(PyString::new_bound(py, key))?;
} else {
list.append(obj.bind(py))?;
}
}
Ok(list)
}
}
#[pymethods]
impl StemmerCore {
/// Compile a model from a gzipped TSV source dictionary.
///
/// * `path` — path to either a gzipped TSV source dictionary
/// (`stem\tvariant1\tvariant2...` per line) OR a compiled `.rxc` trie
/// (Java-interoperable v7 format). The format is auto-detected.
/// * `backward` — BACKWARD traversal (all languages except the
/// right-to-left fa/he/yi, which use FORWARD). Ignored for compiled input
/// (baked into the file).
/// * `store_original` — map each canonical stem to the no-op patch so the
/// stem itself is recognised. Ignored for compiled input.
#[new]
#[pyo3(signature = (path, backward=true, store_original=true, lowercase=true, cache_size=10_000))]
fn new(
path: &str,
backward: bool,
store_original: bool,
lowercase: bool,
cache_size: usize,
) -> PyResult<Self> {
let raw =
fs::read(path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
let decompressed = decompress_or_raw(&raw);
// Auto-detect: a compiled v7 trie starts with the stream magic; anything
// else is a textual TSV dictionary compiled here in Rust.
let trie = if serial::is_v7_stream(&decompressed) {
serial::read_stream(&decompressed)
.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?
} else {
// Dictionary keys are always lowercased at build time (canonical
// form). `lowercase` controls whether lookups lowercase the input at
// runtime; set it False for already-lowercased input.
let text = String::from_utf8_lossy(&decompressed);
let entries = dict::parse_text(&text, true);
builder::build_trie_from_dict(&entries, backward, store_original, lowercase)
};
let cache = if cache_size > 0 {
// Keep PyStemmer's default entry limit without charging every
// Stemmer instance for 10,000 buckets before its first lookup.
Some(Mutex::new(HashMap::new()))
} else {
None
};
Ok(StemmerCore {
trie: Arc::new(trie),
cache,
cache_cap: cache_size,
})
}
fn stem(&self, py: Python<'_>, word: &str) -> Py<PyAny> {
self.stem_cached(
py,
word,
&mut Vec::new(),
&mut Vec::new(),
&mut String::new(),
)
}
/// PyStemmer-compatible scalar API. An unrecognized word is returned
/// unchanged instead of producing None.
#[pyo3(name = "stemWord")]
fn stem_word(&self, py: Python<'_>, word: &str) -> Py<PyAny> {
let obj = self.stem_cached(
py,
word,
&mut Vec::new(),
&mut Vec::new(),
&mut String::new(),
);
if obj.bind(py).is_none() {
PyString::new_bound(py, word).into_any().unbind()
} else {
obj
}
}
fn stem_batch<'py>(
&self,
py: Python<'py>,
words: Vec<PyBackedStr>,
) -> PyResult<Bound<'py, PyList>> {
self.stem_batch_impl(py, &words, false)
}
/// PyStemmer-compatible batch API. Unrecognized words keep their position
/// in the result and are returned unchanged.
#[pyo3(name = "stemWords")]
fn stem_words<'py>(
&self,
py: Python<'py>,
words: Vec<PyBackedStr>,
) -> PyResult<Bound<'py, PyList>> {
self.stem_batch_impl(py, &words, true)
}
fn stem_all(&self, word: &str) -> Vec<String> {
self.trie.stem_all(word)
}
/// Diagnostic: full batch round-trip (marshal input, allocate one String
/// per word, build the result list) with NO stemming. Measures the
/// irreducible Python<->Rust boundary + string-allocation floor.
fn _echo_batch(&self, words: Vec<PyBackedStr>) -> Vec<Option<String>> {
words.iter().map(|w| Some(w.to_string())).collect()
}
/// Diagnostic: pure input marshalling (sum of byte lengths), no stemming,
/// no output strings, no result list.
fn _len_batch(&self, words: Vec<PyBackedStr>) -> u64 {
words.iter().map(|w| w.len() as u64).sum()
}
/// Diagnostic: normalize + UTF-16 encode only.
fn _encode_batch(&self, words: Vec<PyBackedStr>) -> u64 {
let mut key_buf = Vec::new();
words
.iter()
.map(|w| self.trie.bench_encode(w, &mut key_buf) as u64)
.sum()
}
/// Diagnostic: normalize + encode + trie walk (no patch apply).
fn _encodefind_batch(&self, words: Vec<PyBackedStr>) -> u64 {
let mut key_buf = Vec::new();
let mut acc = 0u64;
for w in &words {
if self.trie.bench_find(w, &mut key_buf) {
acc += 1;
}
}
acc
}
/// Diagnostic: full stemming algorithm (normalize + UTF-16 encode + trie
/// walk + patch apply) but returning only the summed stem length — no
/// per-word output String and no Python result list.
fn _stem_lengths_batch(&self, words: Vec<PyBackedStr>) -> u64 {
let mut key_buf = Vec::new();
let mut out_buf = Vec::new();
let mut acc = 0u64;
for w in &words {
if let Some(n) = self.trie.stem_len_into(w, &mut key_buf, &mut out_buf) {
acc += n as u64;
}
}
acc
}
fn stem_all_batch(&self, words: Vec<PyBackedStr>) -> Vec<Vec<String>> {
words.iter().map(|w| self.trie.stem_all(w)).collect()
}
}
/// Compile a gzipped/plain TSV source dictionary into a Java-interoperable
/// compiled trie file (v7 format), so it can be loaded instantly later.
///
/// * `source_path` — path to a `stemmer.gz` (or plain TSV) source dictionary.
/// * `out_path` — destination compiled file (conventionally `*.rxc`).
/// * `backward` / `store_original` / `lowercase` — same meaning as the
/// `Stemmer` constructor; baked into the compiled file.
#[pyfunction]
#[pyo3(signature = (source_path, out_path, backward=true, store_original=true, lowercase=true))]
fn compile(
source_path: &str,
out_path: &str,
backward: bool,
store_original: bool,
lowercase: bool,
) -> PyResult<()> {
let raw =
fs::read(source_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
let decompressed = decompress_or_raw(&raw);
if serial::is_v7_stream(&decompressed) {
return Err(pyo3::exceptions::PyValueError::new_err(
"source is already a compiled trie",
));
}
let text = String::from_utf8_lossy(&decompressed);
let entries = dict::parse_text(&text, true);
let frozen = builder::build_frozen(&entries, backward, store_original);
let metadata = builder::metadata_for(backward, lowercase);
let bytes = serial::write_v7(&frozen, &metadata)
.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
fs::write(out_path, bytes).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
Ok(())
}
#[pymodule]
fn _radixor(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<StemmerCore>()?;
m.add_function(wrap_pyfunction!(compile, m)?)?;
Ok(())
}

494
python/src/patch.rs Normal file
View File

@@ -0,0 +1,494 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#[derive(Debug, Clone)]
pub enum PatchCommand {
Preserve,
DeleteSuffix(usize),
DeletePrefix(usize),
AppendChar(u16),
PrependChar(u16),
ReplaceLastChar(u16),
ReplaceFirstChar(u16),
BackwardCompound {
opcodes: Vec<u8>,
operands: Vec<u32>,
length_delta: i32,
min_len: usize,
},
ForwardCompound {
opcodes: Vec<u8>,
operands: Vec<u32>,
length_delta: i32,
min_len: usize,
},
}
const SKIP: u8 = b'-';
const DELETE: u8 = b'D';
const INSERT: u8 = b'I';
const REPLACE: u8 = b'R';
const NOOP: u8 = b'N';
fn decode_count(arg: u16) -> Option<usize> {
if arg < b'a' as u16 {
return None;
}
Some((arg - b'a' as u16) as usize + 1)
}
fn compile_operand(opcode: u8, arg: u16) -> Option<u32> {
match opcode {
SKIP | DELETE => {
let count = decode_count(arg)?;
if count < 1 {
None
} else {
Some(count as u32)
}
}
INSERT | REPLACE => Some(arg as u32),
NOOP => {
if arg == b'a' as u16 {
None
} else {
panic!("Invalid NOOP arg")
}
}
_ => panic!("Unknown opcode: {}", opcode as char),
}
}
fn length_delta(opcodes: &[u8], operands: &[u32]) -> i32 {
let mut delta: i32 = 0;
for (i, &op) in opcodes.iter().enumerate() {
match op {
DELETE => delta -= operands[i] as i32,
INSERT => delta += 1,
_ => {}
}
}
delta
}
fn backward_min_len(opcodes: &[u8], operands: &[u32]) -> usize {
let mut min_len: usize = 0;
let mut consumed_from_end: usize = 0;
for (i, &op) in opcodes.iter().enumerate() {
let operand = operands[i] as usize;
match op {
SKIP => consumed_from_end += operand,
DELETE => {
min_len = min_len.max(consumed_from_end + operand);
consumed_from_end += operand;
}
INSERT => {
min_len = min_len.max(consumed_from_end);
}
REPLACE => {
min_len = min_len.max(consumed_from_end + 1);
consumed_from_end += 1;
}
_ => {}
}
}
min_len
}
fn forward_min_len(opcodes: &[u8], operands: &[u32]) -> usize {
let mut min_len: usize = 0;
let mut position: i32 = 0;
let mut len_delta: i32 = 0;
for (i, &op) in opcodes.iter().enumerate() {
let operand = operands[i] as i32;
match op {
SKIP => position += operand,
DELETE => {
let needed = (position + operand - len_delta).max(0) as usize;
min_len = min_len.max(needed);
len_delta -= operand;
}
INSERT => {
let needed = (position - len_delta).max(0) as usize;
min_len = min_len.max(needed);
len_delta += 1;
position += 1;
}
REPLACE => {
let needed = (position + 1 - len_delta).max(0) as usize;
min_len = min_len.max(needed);
position += 1;
}
_ => {}
}
}
min_len
}
impl PatchCommand {
pub fn parse(patch: &str, backward: bool) -> Self {
let chars: Vec<u16> = patch.encode_utf16().collect();
let len = chars.len();
if len == 0 || len & 1 != 0 {
return PatchCommand::Preserve;
}
if len == 2 {
let opcode = chars[0] as u8;
let arg = chars[1];
return Self::compile_single(opcode, arg, backward);
}
let op_count = len / 2;
let mut opcodes = Vec::with_capacity(op_count);
let mut operands = Vec::with_capacity(op_count);
for i in 0..op_count {
let opcode = chars[i * 2] as u8;
let arg = chars[i * 2 + 1];
match compile_operand(opcode, arg) {
None => return PatchCommand::Preserve,
Some(operand) => {
opcodes.push(opcode);
operands.push(operand);
}
}
}
let ld = length_delta(&opcodes, &operands);
if backward {
let min_len = backward_min_len(&opcodes, &operands);
PatchCommand::BackwardCompound {
opcodes,
operands,
length_delta: ld,
min_len,
}
} else {
let min_len = forward_min_len(&opcodes, &operands);
PatchCommand::ForwardCompound {
opcodes,
operands,
length_delta: ld,
min_len,
}
}
}
fn compile_single(opcode: u8, arg: u16, backward: bool) -> Self {
match opcode {
DELETE => {
let count = match decode_count(arg) {
Some(c) if c >= 1 => c,
_ => return PatchCommand::Preserve,
};
if backward {
PatchCommand::DeleteSuffix(count)
} else {
PatchCommand::DeletePrefix(count)
}
}
INSERT => {
if backward {
PatchCommand::AppendChar(arg)
} else {
PatchCommand::PrependChar(arg)
}
}
REPLACE => {
if backward {
PatchCommand::ReplaceLastChar(arg)
} else {
PatchCommand::ReplaceFirstChar(arg)
}
}
SKIP | NOOP => PatchCommand::Preserve,
_ => panic!("Unknown opcode: {}", opcode as char),
}
}
fn computed_length(&self, src_len: usize) -> usize {
let (ld, min_len) = match self {
PatchCommand::Preserve => (0i32, 0usize),
PatchCommand::DeleteSuffix(n) | PatchCommand::DeletePrefix(n) => (-(*n as i32), 0),
PatchCommand::AppendChar(_) | PatchCommand::PrependChar(_) => (1, 0),
PatchCommand::ReplaceLastChar(_) | PatchCommand::ReplaceFirstChar(_) => (0, 1),
PatchCommand::BackwardCompound {
length_delta,
min_len,
..
} => (*length_delta, *min_len),
PatchCommand::ForwardCompound {
length_delta,
min_len,
..
} => (*length_delta, *min_len),
};
if src_len < min_len {
return src_len;
}
let applied = src_len as i32 + ld;
if applied < 1 {
src_len
} else {
applied as usize
}
}
pub fn apply(&self, source: &[u16]) -> Vec<u16> {
let mut out = Vec::new();
self.apply_into(source, &mut out);
out
}
/// Apply the patch into a caller-owned buffer, avoiding a per-call
/// allocation on the hot path. `out` is cleared and overwritten.
pub fn apply_into(&self, source: &[u16], out: &mut Vec<u16>) {
let src_len = source.len();
let out_len = self.computed_length(src_len);
out.clear();
match self {
PatchCommand::Preserve => out.extend_from_slice(source),
PatchCommand::DeleteSuffix(_) => {
if out_len < src_len {
out.extend_from_slice(&source[..out_len]);
} else {
out.extend_from_slice(source);
}
}
PatchCommand::DeletePrefix(n) => {
if out_len < src_len {
out.extend_from_slice(&source[*n..]);
} else {
out.extend_from_slice(source);
}
}
PatchCommand::AppendChar(ch) => {
out.extend_from_slice(source);
out.push(*ch);
}
PatchCommand::PrependChar(ch) => {
out.push(*ch);
out.extend_from_slice(source);
}
PatchCommand::ReplaceLastChar(ch) => {
out.extend_from_slice(source);
if src_len != 0 {
let l = out.len();
out[l - 1] = *ch;
}
}
PatchCommand::ReplaceFirstChar(ch) => {
out.extend_from_slice(source);
if src_len != 0 {
out[0] = *ch;
}
}
PatchCommand::BackwardCompound {
opcodes, operands, ..
} => {
if src_len < self.min_len_for_compound() || out_len < 1 {
out.extend_from_slice(source);
} else {
apply_backward_into(opcodes, operands, source, out_len, out);
}
}
PatchCommand::ForwardCompound {
opcodes, operands, ..
} => {
if src_len < self.min_len_for_compound() || out_len < 1 {
out.extend_from_slice(source);
} else {
apply_forward_into(opcodes, operands, source, out_len, out);
}
}
}
}
fn min_len_for_compound(&self) -> usize {
match self {
PatchCommand::BackwardCompound { min_len, .. } => *min_len,
PatchCommand::ForwardCompound { min_len, .. } => *min_len,
_ => 0,
}
}
}
fn fill_with_source(out: &mut Vec<u16>, source: &[u16]) {
out.clear();
out.extend_from_slice(source);
}
fn apply_backward_into(
opcodes: &[u8],
operands: &[u32],
source: &[u16],
produced_len: usize,
out: &mut Vec<u16>,
) {
let src_len = source.len();
out.clear();
out.resize(produced_len, 0);
let mut current_len = src_len as i32;
let mut position = src_len as i32 - 1;
let mut src_end = src_len as i32;
let mut out_end = produced_len as i32;
for (i, &op) in opcodes.iter().enumerate() {
let operand = operands[i] as i32;
match op {
SKIP => {
let skip = operand.min(src_end);
src_end -= skip;
out_end -= skip;
if out_end < 0 {
return fill_with_source(out, source);
}
let s = src_end as usize;
let o = out_end as usize;
out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]);
position = position - operand + 1;
}
DELETE => {
let del_end_excl = position + 1;
position -= operand - 1;
if position < 0 || position > current_len || position > del_end_excl {
return fill_with_source(out, source);
}
let deleted = (del_end_excl.min(current_len) - position) as i32;
if src_end < deleted {
return fill_with_source(out, source);
}
src_end -= deleted;
current_len -= deleted;
}
INSERT => {
if position < -1 || position >= current_len || out_end <= 0 {
return fill_with_source(out, source);
}
out_end -= 1;
out[out_end as usize] = operand as u16;
current_len += 1;
position += 1;
}
REPLACE => {
if position < 0 || position >= current_len || src_end <= 0 || out_end <= 0 {
return fill_with_source(out, source);
}
src_end -= 1;
out_end -= 1;
out[out_end as usize] = operand as u16;
}
_ => return fill_with_source(out, source),
}
position -= 1;
}
if src_end != out_end {
return fill_with_source(out, source);
}
let prefix_len = src_end as usize;
out[..prefix_len].copy_from_slice(&source[..prefix_len]);
}
fn apply_forward_into(
opcodes: &[u8],
operands: &[u32],
source: &[u16],
produced_len: usize,
out: &mut Vec<u16>,
) {
let src_len = source.len();
out.clear();
out.resize(produced_len, 0);
let mut current_len = src_len as i32;
let mut position: i32 = 0;
let mut src_idx: i32 = 0;
let mut out_idx: i32 = 0;
for (i, &op) in opcodes.iter().enumerate() {
let operand = operands[i] as i32;
match op {
SKIP => {
let skip = operand.min(src_len as i32 - src_idx);
let s = src_idx as usize;
let o = out_idx as usize;
out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]);
src_idx += skip;
out_idx += skip;
position = position + operand - 1;
}
DELETE => {
if position < 0 || position > current_len {
return fill_with_source(out, source);
}
let del_len = operand.min(current_len - position);
if src_idx + del_len > src_len as i32 {
return fill_with_source(out, source);
}
src_idx += del_len;
current_len -= del_len;
position -= 1;
}
INSERT => {
if position < 0 || position > current_len || out_idx >= produced_len as i32 {
return fill_with_source(out, source);
}
out[out_idx as usize] = operand as u16;
out_idx += 1;
current_len += 1;
}
REPLACE => {
if position < 0
|| position >= current_len
|| src_idx >= src_len as i32
|| out_idx >= produced_len as i32
{
return fill_with_source(out, source);
}
src_idx += 1;
out[out_idx as usize] = operand as u16;
out_idx += 1;
}
_ => return fill_with_source(out, source),
}
position += 1;
}
let remaining = (src_len as i32 - src_idx) as usize;
if remaining > produced_len - out_idx as usize {
return fill_with_source(out, source);
}
let o = out_idx as usize;
let s = src_idx as usize;
out[o..o + remaining].copy_from_slice(&source[s..s + remaining]);
if out_idx as usize + remaining != produced_len {
fill_with_source(out, source);
}
}

450
python/src/serial.rs Normal file
View File

@@ -0,0 +1,450 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// Java-interoperable compiled-trie binary I/O ("v7" stream), matching
// org.egothor.stemmer.StemmerPatchTrieBinaryIO / FrequencyTrie.writeTo/readFrom.
//
// File layout = gzip( big-endian Java DataOutputStream stream ):
// i32 STREAM_MAGIC=0x45475452 ; i32 STREAM_VERSION=7
// i32 nodeCount ; i32 rootId(=0)
// writeUTF(metadata.toTextBlock()) // Java modified UTF-8
// i32 valueCount ; valueCount x writeUTF(patch) // value dictionary
// per node id 0..nodeCount-1:
// u8 acceptsRemainingInput
// i32 edgeCount ; edgeCount x { u16 edgeLabel ; i32 childId }
// i32 valueCount ; valueCount x { i32 valueId ; i32 count }
//
// The outer gzip framing (headers/mtime) is not byte-identical across Java and
// Rust, but the INNER stream is, and both directions gunzip+parse each other.
use std::collections::HashMap;
use std::io::{self, Read, Write};
use std::sync::Arc;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use crate::builder::{FrozenTrie, MAX_DENSE_SPAN};
use crate::patch::PatchCommand;
use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata};
const STREAM_MAGIC: i32 = 0x4547_5452;
const STREAM_VERSION: i32 = 7;
// Big-endian writer helpers matching Java DataOutputStream.
fn put_i32(out: &mut Vec<u8>, v: i32) {
out.extend_from_slice(&v.to_be_bytes());
}
fn put_u16(out: &mut Vec<u8>, v: u16) {
out.extend_from_slice(&v.to_be_bytes());
}
/// Java DataOutputStream.writeUTF: u16 big-endian byte length + modified UTF-8.
fn put_java_utf(out: &mut Vec<u8>, s: &str) -> io::Result<()> {
let mut bytes: Vec<u8> = Vec::with_capacity(s.len());
for u in s.encode_utf16() {
if (0x0001..=0x007F).contains(&u) {
bytes.push(u as u8);
} else if u == 0 || (0x0080..=0x07FF).contains(&u) {
bytes.push(0xC0 | ((u >> 6) as u8 & 0x1F));
bytes.push(0x80 | (u as u8 & 0x3F));
} else {
bytes.push(0xE0 | ((u >> 12) as u8 & 0x0F));
bytes.push(0x80 | ((u >> 6) as u8 & 0x3F));
bytes.push(0x80 | (u as u8 & 0x3F));
}
}
if bytes.len() > 0xFFFF {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"string too long for Java modified UTF-8",
));
}
put_u16(out, bytes.len() as u16);
out.extend_from_slice(&bytes);
Ok(())
}
// Metadata text block, byte-identical to TrieMetadata.toTextBlock.
fn text_block(meta: &TrieMetadata) -> String {
let forward = matches!(meta.traversal, TraversalDirection::Forward);
let case = match meta.case_mode {
CaseMode::LowercaseWithLocaleRoot => "LOWERCASE_WITH_LOCALE_ROOT",
CaseMode::AsIs => "AS_IS",
};
let diac = match meta.diacritic_mode {
DiacriticMode::AsIs => "AS_IS",
DiacriticMode::Remove => "REMOVE",
};
let mut s = String::with_capacity(256);
s.push_str("radixor.metadata.v1\n");
s.push_str("formatVersion=7\n");
s.push_str(if forward {
"traversalDirection=FORWARD\n"
} else {
"traversalDirection=BACKWARD\n"
});
s.push_str(if forward {
"rightToLeft=true\n"
} else {
"rightToLeft=false\n"
});
s.push_str("reductionMode=MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS\n");
s.push_str("dominantWinnerMinPercent=75\n");
s.push_str("dominantWinnerOverSecondRatio=3\n");
s.push_str("contractUniformSubtrees=true\n");
s.push_str(&format!("diacriticProcessingMode={}\n", diac));
s.push_str(&format!("caseProcessingMode={}\n", case));
s
}
/// Serialize the frozen trie to the inner (uncompressed) v7 stream.
fn write_stream(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result<Vec<u8>> {
let node_count = frozen.accepts.len();
let mut out = Vec::with_capacity(1024 + frozen.edge_labels.len() * 6);
put_i32(&mut out, STREAM_MAGIC);
put_i32(&mut out, STREAM_VERSION);
put_i32(&mut out, node_count as i32);
put_i32(&mut out, 0); // rootId
put_java_utf(&mut out, &text_block(meta))?;
// Value dictionary: distinct patch strings in first-occurrence order across
// nodes(id) x values(local) — frozen.value_strings is already in that order.
let mut value_id: HashMap<&str, i32> = HashMap::new();
let mut distinct: Vec<&str> = Vec::new();
for s in &frozen.value_strings {
if !value_id.contains_key(s.as_str()) {
value_id.insert(s.as_str(), distinct.len() as i32);
distinct.push(s.as_str());
}
}
put_i32(&mut out, distinct.len() as i32);
for s in &distinct {
put_java_utf(&mut out, s)?;
}
for node in 0..node_count {
out.push(if frozen.accepts[node] { 1 } else { 0 });
let elo = frozen.edge_start[node] as usize;
let ehi = frozen.edge_start[node + 1] as usize;
put_i32(&mut out, (ehi - elo) as i32);
for k in elo..ehi {
put_u16(&mut out, frozen.edge_labels[k]);
put_i32(&mut out, frozen.edge_targets[k] as i32);
}
let vlo = frozen.value_start[node] as usize;
let vhi = frozen.value_start[node + 1] as usize;
put_i32(&mut out, (vhi - vlo) as i32);
for k in vlo..vhi {
let id = value_id[frozen.value_strings[k].as_str()];
put_i32(&mut out, id);
put_i32(&mut out, frozen.value_counts[k]);
}
}
Ok(out)
}
/// Serialize the frozen trie to a gzip-compressed v7 file image.
pub(crate) fn write_v7(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result<Vec<u8>> {
let stream = write_stream(frozen, meta)?;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&stream)?;
encoder.finish()
}
// Compiled-stream reader.
struct Reader<'a> {
data: &'a [u8],
pos: usize,
}
impl<'a> Reader<'a> {
fn new(data: &'a [u8]) -> Self {
Reader { data, pos: 0 }
}
fn take(&mut self, n: usize) -> io::Result<&'a [u8]> {
if self.pos + n > self.data.len() {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected end of trie stream",
));
}
let slice = &self.data[self.pos..self.pos + n];
self.pos += n;
Ok(slice)
}
fn i32(&mut self) -> io::Result<i32> {
let b = self.take(4)?;
Ok(i32::from_be_bytes([b[0], b[1], b[2], b[3]]))
}
fn u16(&mut self) -> io::Result<u16> {
let b = self.take(2)?;
Ok(u16::from_be_bytes([b[0], b[1]]))
}
fn u8(&mut self) -> io::Result<u8> {
Ok(self.take(1)?[0])
}
fn java_utf(&mut self) -> io::Result<String> {
let len = self.u16()? as usize;
let bytes = self.take(len)?;
decode_java_utf(bytes)
}
}
fn decode_java_utf(bytes: &[u8]) -> io::Result<String> {
let mut units: Vec<u16> = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
let b = bytes[i];
if b & 0x80 == 0 {
units.push(b as u16);
i += 1;
} else if b & 0xE0 == 0xC0 {
if i + 1 >= bytes.len() {
return Err(malformed());
}
let b1 = bytes[i + 1];
units.push((((b as u16 & 0x1F) << 6) | (b1 as u16 & 0x3F)) as u16);
i += 2;
} else if b & 0xF0 == 0xE0 {
if i + 2 >= bytes.len() {
return Err(malformed());
}
let b1 = bytes[i + 1];
let b2 = bytes[i + 2];
units.push(((b as u16 & 0x0F) << 12) | ((b1 as u16 & 0x3F) << 6) | (b2 as u16 & 0x3F));
i += 3;
} else {
return Err(malformed());
}
}
Ok(String::from_utf16_lossy(&units))
}
fn malformed() -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, "malformed modified UTF-8")
}
fn parse_metadata(text: &str) -> TrieMetadata {
let mut traversal = TraversalDirection::Backward;
let mut case_mode = CaseMode::LowercaseWithLocaleRoot;
let mut diacritic_mode = DiacriticMode::AsIs;
for line in text.lines() {
if let Some((key, value)) = line.split_once('=') {
match key {
"traversalDirection" => {
traversal = if value == "FORWARD" {
TraversalDirection::Forward
} else {
TraversalDirection::Backward
};
}
"caseProcessingMode" => {
case_mode = if value == "AS_IS" {
CaseMode::AsIs
} else {
CaseMode::LowercaseWithLocaleRoot
};
}
"diacriticProcessingMode" => {
diacritic_mode = if value == "REMOVE" {
DiacriticMode::Remove
} else {
DiacriticMode::AsIs
};
}
_ => {}
}
}
}
TrieMetadata {
traversal,
case_mode,
diacritic_mode,
}
}
/// Rebuild dense direct-index tables from the CSR edges (same policy as freeze).
fn build_dense(
edge_start: &[u32],
edge_labels: &[u16],
edge_targets: &[u32],
) -> (Vec<u32>, Vec<u16>, Vec<u32>) {
let node_count = edge_start.len() - 1;
let mut dense_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut dense_base: Vec<u16> = Vec::with_capacity(node_count);
let mut dense_targets: Vec<u32> = Vec::new();
dense_start.push(0);
for node in 0..node_count {
let lo = edge_start[node] as usize;
let hi = edge_start[node + 1] as usize;
let count = hi - lo;
let mut dense = false;
if count >= 2 {
let first = edge_labels[lo] as usize;
let last = edge_labels[hi - 1] as usize;
let span = last - first + 1;
if span <= MAX_DENSE_SPAN {
let base = edge_labels[lo];
let seg = dense_targets.len();
dense_targets.resize(seg + span, 0);
for k in lo..hi {
dense_targets[seg + (edge_labels[k] - base) as usize] = edge_targets[k] + 1;
}
dense_base.push(base);
dense_start.push(dense_targets.len() as u32);
dense = true;
}
}
if !dense {
dense_base.push(0);
dense_start.push(dense_targets.len() as u32);
}
}
(dense_start, dense_base, dense_targets)
}
/// Read a gzip-compressed Java v7 compiled-trie image into a runtime trie.
#[allow(dead_code)] // convenience wrapper; lib.rs decompresses then calls read_stream
pub(crate) fn read_v7(gz_bytes: &[u8]) -> io::Result<FrequencyTrie> {
let mut data = Vec::new();
GzDecoder::new(gz_bytes).read_to_end(&mut data)?;
read_stream(&data)
}
/// Whether `decompressed` (an already-gunzipped byte stream) is a v7 trie image.
pub(crate) fn is_v7_stream(decompressed: &[u8]) -> bool {
decompressed.len() >= 4
&& i32::from_be_bytes([
decompressed[0],
decompressed[1],
decompressed[2],
decompressed[3],
]) == STREAM_MAGIC
}
/// Parse the inner (uncompressed) v7 stream into a runtime trie.
pub(crate) fn read_stream(data: &[u8]) -> io::Result<FrequencyTrie> {
let mut r = Reader::new(data);
if r.i32()? != STREAM_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"bad trie stream magic",
));
}
let version = r.i32()?;
if version != STREAM_VERSION {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported trie stream version {version} (expected {STREAM_VERSION})"),
));
}
let node_count = r.i32()? as usize;
let root_id = r.i32()?;
if root_id != 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"unsupported non-zero root node id",
));
}
let metadata = parse_metadata(&r.java_utf()?);
let backward = matches!(metadata.traversal, TraversalDirection::Backward);
let value_table_len = r.i32()? as usize;
let mut value_table: Vec<Arc<PatchCommand>> = Vec::with_capacity(value_table_len);
for _ in 0..value_table_len {
let patch = r.java_utf()?;
value_table.push(Arc::new(PatchCommand::parse(&patch, backward)));
}
let mut edge_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut edge_labels: Vec<u16> = Vec::new();
let mut edge_targets: Vec<u32> = Vec::new();
let mut accepts: Vec<bool> = Vec::with_capacity(node_count);
let mut value_start: Vec<u32> = Vec::with_capacity(node_count + 1);
let mut values: Vec<Arc<PatchCommand>> = Vec::new();
edge_start.push(0);
value_start.push(0);
for _ in 0..node_count {
accepts.push(r.u8()? != 0);
let edge_count = r.i32()? as usize;
for _ in 0..edge_count {
let label = r.u16()?;
let child = r.i32()? as u32;
edge_labels.push(label);
edge_targets.push(child);
}
edge_start.push(edge_labels.len() as u32);
let value_count = r.i32()? as usize;
for _ in 0..value_count {
let value_id = r.i32()? as usize;
let _count = r.i32()?; // frequency: not used at runtime
if value_id >= value_table.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"value id out of range",
));
}
values.push(Arc::clone(&value_table[value_id]));
}
value_start.push(values.len() as u32);
}
let (dense_start, dense_base, dense_targets) =
build_dense(&edge_start, &edge_labels, &edge_targets);
Ok(FrequencyTrie::new(
edge_start,
edge_labels,
edge_targets,
accepts,
value_start,
values,
dense_start,
dense_base,
dense_targets,
metadata,
))
}

294
python/src/trie.rs Normal file
View File

@@ -0,0 +1,294 @@
// Copyright (C) 2026, Leo Galambos
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
use crate::patch::PatchCommand;
use std::borrow::Cow;
use std::sync::Arc;
use unicode_general_category::{get_general_category, GeneralCategory};
use unicode_normalization::UnicodeNormalization;
#[derive(Debug, Clone)]
pub enum TraversalDirection {
Backward,
Forward,
}
#[derive(Debug, Clone)]
pub enum CaseMode {
LowercaseWithLocaleRoot,
AsIs,
}
#[derive(Debug, Clone)]
pub enum DiacriticMode {
AsIs,
Remove,
}
#[derive(Debug, Clone)]
pub struct TrieMetadata {
pub traversal: TraversalDirection,
pub case_mode: CaseMode,
pub diacritic_mode: DiacriticMode,
}
/// Compiled patch-command trie in a flat, cache-friendly CSR layout.
///
/// Instead of a graph of heap-allocated, reference-counted nodes (which forces
/// a pointer chase and a likely cache miss at every character step), the whole
/// trie is stored as a handful of contiguous arrays indexed by node id:
///
/// * `edge_start[i] .. edge_start[i+1]` slices `edge_labels` / `edge_targets`
/// for node `i` (labels sorted ascending, so child lookup is a binary search
/// over a contiguous, cache-hot slice — no pointer chasing, no atomics),
/// * `accepts[i]` marks a contracted accepting leaf,
/// * `value_start[i] .. value_start[i+1]` slices `values` (best value first).
///
/// Node 0 is the root. Shared (deduplicated) subtrees simply reference the same
/// node id, so structural sharing from reduction is preserved without `Arc`.
pub struct FrequencyTrie {
edge_start: Vec<u32>,
edge_labels: Vec<u16>,
edge_targets: Vec<u32>,
accepts: Vec<bool>,
value_start: Vec<u32>,
values: Vec<Arc<PatchCommand>>,
// Adaptive child lookup (mirrors the Java CompiledNode fanout strategy):
// high-fanout nodes whose child labels span a small contiguous range get a
// dense direct-index table (O(1) child access); sparse nodes fall back to
// binary search over `edge_labels`. A node `i` is dense iff
// `dense_start[i+1] > dense_start[i]`; then `dense_targets[dense_start[i] +
// (label - dense_base[i])]` holds `child_id + 1` (0 = no such edge).
dense_start: Vec<u32>,
dense_base: Vec<u16>,
dense_targets: Vec<u32>,
pub metadata: TrieMetadata,
}
impl FrequencyTrie {
#[allow(clippy::too_many_arguments)]
pub fn new(
edge_start: Vec<u32>,
edge_labels: Vec<u16>,
edge_targets: Vec<u32>,
accepts: Vec<bool>,
value_start: Vec<u32>,
values: Vec<Arc<PatchCommand>>,
dense_start: Vec<u32>,
dense_base: Vec<u16>,
dense_targets: Vec<u32>,
metadata: TrieMetadata,
) -> Self {
FrequencyTrie {
edge_start,
edge_labels,
edge_targets,
accepts,
value_start,
values,
dense_start,
dense_base,
dense_targets,
metadata,
}
}
/// Normalize a lookup key (used by the rare diacritic-removal path and by
/// `stem_all`). Borrows the input when no transformation is needed.
fn normalize_key<'a>(&self, word: &'a str) -> Cow<'a, str> {
let lowered: Cow<'a, str> =
if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot)
&& word.chars().any(|c| c.is_uppercase())
{
Cow::Owned(word.to_lowercase())
} else {
Cow::Borrowed(word)
};
if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) {
Cow::Owned(strip_diacritics(&lowered))
} else {
lowered
}
}
/// Encode the normalized lookup key into `key_buf` in a single pass over the
/// input: lowercasing (when configured) is folded into the UTF-16 encoding
/// so the UTF-8 input is decoded only once and no intermediate `String` is
/// allocated. The diacritic-removal path (unused by the bundled models)
/// falls back to the general `normalize_key`.
#[inline]
fn encode_key(&self, word: &str, key_buf: &mut Vec<u16>) {
key_buf.clear();
if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) {
let normalized = self.normalize_key(word);
key_buf.extend(normalized.encode_utf16());
return;
}
if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot) {
let mut unit = [0u16; 2];
for c in word.chars() {
if c.is_ascii() {
// ASCII fast path: lowercasing requires a single branch.
key_buf.push(c.to_ascii_lowercase() as u16);
} else if c.is_lowercase() {
// Already lowercase (e.g. lowercase Cyrillic/Greek): encode
// directly and skip the costly Unicode special-casing.
key_buf.extend_from_slice(c.encode_utf16(&mut unit));
} else {
for lc in c.to_lowercase() {
key_buf.extend_from_slice(lc.encode_utf16(&mut unit));
}
}
}
} else {
key_buf.extend(word.encode_utf16());
}
}
/// Find the child of `node` on `label` via binary search over the node's
/// contiguous, ascending edge-label slice. Uses unchecked indexing on
/// provably in-range offsets to drop bounds checks from the hot loop.
#[inline]
fn child(&self, node: usize, label: u16) -> Option<usize> {
// Dense high-fanout node: O(1) direct index.
// SAFETY: node and node+1 index dense_start (len = num_nodes+1).
let ds = unsafe { *self.dense_start.get_unchecked(node) } as usize;
let de = unsafe { *self.dense_start.get_unchecked(node + 1) } as usize;
if de > ds {
let base = unsafe { *self.dense_base.get_unchecked(node) };
let idx = label.wrapping_sub(base) as usize;
if idx < de - ds {
// SAFETY: ds + idx < de <= dense_targets.len().
let t = unsafe { *self.dense_targets.get_unchecked(ds + idx) };
if t != 0 {
return Some((t - 1) as usize);
}
}
return None;
}
// Sparse node: binary search over the contiguous ascending edge slice.
// SAFETY: node and node+1 index edge_start (len = num_nodes+1).
let lo = unsafe { *self.edge_start.get_unchecked(node) } as usize;
let hi = unsafe { *self.edge_start.get_unchecked(node + 1) } as usize;
// SAFETY: lo <= hi <= edge_labels.len() by construction.
let labels = unsafe { self.edge_labels.get_unchecked(lo..hi) };
match labels.binary_search(&label) {
// SAFETY: lo+pos < hi <= edge_targets.len().
Ok(pos) => Some(unsafe { *self.edge_targets.get_unchecked(lo + pos) } as usize),
Err(_) => None,
}
}
/// Walk the trie for `key`, returning the accepting/terminal node id.
#[inline]
fn find_node(&self, key: &[u16]) -> Option<usize> {
let mut node = 0usize;
match self.metadata.traversal {
TraversalDirection::Backward => {
for &label in key.iter().rev() {
if unsafe { *self.accepts.get_unchecked(node) } {
return Some(node);
}
node = self.child(node, label)?;
}
}
TraversalDirection::Forward => {
for &label in key.iter() {
if unsafe { *self.accepts.get_unchecked(node) } {
return Some(node);
}
node = self.child(node, label)?;
}
}
}
Some(node)
}
#[inline]
fn preferred_value(&self, node: usize) -> Option<&Arc<PatchCommand>> {
let start = self.value_start[node] as usize;
let end = self.value_start[node + 1] as usize;
if start == end {
None
} else {
Some(&self.values[start])
}
}
/// Stem into caller-owned scratch buffers and return the produced length
/// without allocating an output String. This also supports diagnostics
/// that isolate the algorithm from output-String allocation.
pub fn stem_len_into(
&self,
word: &str,
key_buf: &mut Vec<u16>,
out_buf: &mut Vec<u16>,
) -> Option<usize> {
self.encode_key(word, key_buf);
let node = self.find_node(key_buf)?;
let patch = self.preferred_value(node)?;
patch.apply_into(key_buf, out_buf);
Some(out_buf.len())
}
/// Diagnostic: only normalize + UTF-16 encode the key.
pub fn bench_encode(&self, word: &str, key_buf: &mut Vec<u16>) -> usize {
self.encode_key(word, key_buf);
key_buf.len()
}
/// Diagnostic: normalize + encode + trie walk (no patch apply).
pub fn bench_find(&self, word: &str, key_buf: &mut Vec<u16>) -> bool {
self.encode_key(word, key_buf);
self.find_node(key_buf).is_some()
}
/// Return all stems in frequency order.
pub fn stem_all(&self, word: &str) -> Vec<String> {
let mut key_u16: Vec<u16> = Vec::new();
self.encode_key(word, &mut key_u16);
match self.find_node(&key_u16) {
None => Vec::new(),
Some(node) => {
let start = self.value_start[node] as usize;
let end = self.value_start[node + 1] as usize;
self.values[start..end]
.iter()
.map(|p| String::from_utf16_lossy(&p.apply(&key_u16)))
.collect()
}
}
}
}
pub fn strip_diacritics(s: &str) -> String {
s.nfd()
.filter(|ch| !matches!(get_general_category(*ch), GeneralCategory::NonspacingMark))
.collect()
}

44
python/tests/conftest.py Normal file
View File

@@ -0,0 +1,44 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Make the generated standard resource package available to source tests."""
from __future__ import annotations
import sys
from pathlib import Path
REPOSITORY = Path(__file__).resolve().parents[2]
GENERATED_MODELS_DISTRIBUTION = (
REPOSITORY / "build" / "python" / "generated" / "models-standard"
)
if GENERATED_MODELS_DISTRIBUTION.is_dir():
sys.path.insert(0, str(GENERATED_MODELS_DISTRIBUTION))

93
python/tests/test_distribution_verifier.py Normal file
View File

@@ -0,0 +1,93 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Regression tests for Python release-archive metadata validation."""
from __future__ import annotations
import sys
from email.message import Message
from pathlib import Path
import pytest
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
sys.path.insert(0, str(SCRIPTS))
from build_standard_distribution import ( # noqa: E402
_ignore_build_artifacts,
_validate_generated_project,
)
from verify_distributions import _assert_main_dependency # noqa: E402
def test_standard_model_source_tree_contains_no_generated_payload() -> None:
source = Path(__file__).resolve().parents[1] / "models-standard"
package = source / "radixor_models_standard"
assert not (package / "manifest.json").exists()
assert not list((package / "models").glob("*.rxc"))
assert not list((package / "notices").glob("*/NOTICE-model-data.txt"))
def test_standard_model_build_rejects_source_skeleton() -> None:
source = Path(__file__).resolve().parents[1] / "models-standard"
with pytest.raises(ValueError, match="manifest is missing"):
_validate_generated_project(source)
def test_standard_model_build_ignores_local_build_state() -> None:
names = [
"build",
"dist",
"radixor_models_standard.egg-info",
"__pycache__",
"module.pyc",
"module.pyo",
"manifest.json",
]
assert _ignore_build_artifacts("unused", names) == set(names[:-1])
def test_native_distribution_requires_compatible_standard_models() -> None:
metadata = Message()
metadata["Requires-Dist"] = "radixor-models-standard >=1.0, <2.0"
_assert_main_dependency(metadata, "radixor-4.1.0.tar.gz")
def test_native_distribution_rejects_missing_standard_models() -> None:
metadata = Message()
with pytest.raises(ValueError, match="standard-model dependency"):
_assert_main_dependency(metadata, "radixor-4.1.0.tar.gz")

303
python/tests/test_radixor.py Normal file
View File

@@ -0,0 +1,303 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Acceptance tests for the radixor Python extension.
Run after building the extension:
cd python/
pip install maturin pytest
maturin develop --release
pytest -q
The synthetic tests are self-contained and deterministic (no network, no
bundled data).
"""
from __future__ import annotations
import gzip
import inspect
from pathlib import Path
from radixor import Stemmer
def _write_gz_dict(lines: list[str], tmp_path: Path) -> str:
"""Write a gzipped TSV dictionary into pytest's temporary directory."""
path = tmp_path / "dictionary.gz"
with gzip.open(path, "wt", encoding="utf-8", newline="\n") as gz:
gz.write("\n".join(lines))
return str(path)
# Synthetic, deterministic pipeline tests.
def test_backward_suffix_stemming_roundtrip(tmp_path: Path):
# stem<TAB>variant... ; backward (suffix) stemming.
dict_lines = [
"run\trunning\truns\tran",
"cat\tcats",
"walk\twalking\twalks\twalked",
]
path = _write_gz_dict(dict_lines, tmp_path)
s = Stemmer(path=path, backward=True, store_original=True)
# Every listed variant must stem back to its canonical stem.
assert s.stem("running") == "run"
assert s.stem("runs") == "run"
assert s.stem("ran") == "run"
assert s.stem("cats") == "cat"
assert s.stem("walking") == "walk"
assert s.stem("walked") == "walk"
# store_original: the stem itself is recognised (no-op patch).
assert s.stem("run") == "run"
assert s.stem("cat") == "cat"
def test_store_original_controls_bare_stem_identity(tmp_path: Path):
# With a single rule and store_original=True, the stem maps to itself via
# the no-op patch, and the "cat" vs "cats" terminals carry different values
# so the trie does NOT collapse to a universal rule.
path = _write_gz_dict(["cat\tcats"], tmp_path)
s_keep = Stemmer(path=path, backward=True, store_original=True)
assert s_keep.stem("cats") == "cat"
assert s_keep.stem("cat") == "cat"
# With store_original=False, only the single rule cats->cat is present.
# Radixor's always-on uniform-subtree contraction generalizes that lone
# rule to ALL input (this is the intended generalization behavior), so the
# bare stem is rewritten by the same delete-one-suffix rule.
s_drop = Stemmer(path=path, backward=True, store_original=False)
assert s_drop.stem("cats") == "cat"
assert s_drop.stem("cat") == "ca" # generalized: delete final char
assert s_drop.stem("dogs") == "dog" # rule applies to unseen input too
def test_unknown_word_returns_none(tmp_path: Path):
path = _write_gz_dict(["cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True)
assert s.stem("zzzunknown") is None
def test_pystemmer_scalar_api_returns_original_word_for_unknown(tmp_path: Path):
path = _write_gz_dict(["cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True)
assert s.stemWord("cats") == "cat"
assert s.stemWord("ZzZUnknown") == "ZzZUnknown"
# The original Radixor API keeps its existing missing-value contract.
assert s.stem("ZzZUnknown") is None
def test_pystemmer_batch_api_returns_original_words_for_unknowns(tmp_path: Path):
path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True)
words = ["running", "Nope", "cats", "QzXqZx"]
assert s.stemWords(words) == ["run", "Nope", "cat", "QzXqZx"]
assert s.stem_batch(words) == ["run", None, "cat", None]
def test_pystemmer_batch_cache_does_not_change_original_api(tmp_path: Path):
path = _write_gz_dict(["cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True, cache_size=100)
assert s.stemWords(["Unknown", "cats", "Unknown"]) == ["Unknown", "cat", "Unknown"]
assert s.stem_batch(["Unknown", "cats", "Unknown"]) == [None, "cat", None]
def test_wrapper_forwards_default_cache_size_and_zero_opt_out(monkeypatch):
import radixor
constructor_calls = []
class RecordingStemmerCore:
def __init__(self, *args):
constructor_calls.append(args)
monkeypatch.setattr(radixor, "StemmerCore", RecordingStemmerCore)
radixor.Stemmer(path="model.rxc")
radixor.Stemmer(path="model.rxc", cache_size=0)
assert constructor_calls[0][-1] == 10_000
assert constructor_calls[1][-1] == 0
def test_native_constructor_default_cache_size():
from radixor._radixor import StemmerCore
assert inspect.signature(StemmerCore).parameters["cache_size"].default == 10_000
def test_pystemmer_language_name_alias():
import radixor as StemmerModule
# Only the dependency/import line changes from PyStemmer's conventional
# ``import Stemmer; Stemmer.Stemmer("english")`` usage.
s = StemmerModule.Stemmer("english")
assert s.stemWord("running") == "run"
assert s.stemWords(["running", "unknown_word"]) == ["run", "unknown_word"]
def test_case_is_lowercased(tmp_path: Path):
path = _write_gz_dict(["cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True)
assert s.stem("CATS") == "cat"
assert s.stem("Cats") == "cat"
def test_batch_matches_scalar(tmp_path: Path):
path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True)
words = ["running", "runs", "cats", "nope", "run"]
assert s.stem_batch(words) == [s.stem(w) for w in words]
def test_compile_roundtrip_matches_from_text(tmp_path: Path):
import os
import radixor
dict_lines = [
"run\trunning\truns\tran",
"cat\tcats",
"walk\twalking\twalks\twalked",
]
src = _write_gz_dict(dict_lines, tmp_path)
out = src + ".rxc"
radixor.compile(src, out, backward=True)
from_text = Stemmer(path=src, backward=True)
from_compiled = Stemmer(compiled=out)
words = [
"running",
"runs",
"ran",
"cats",
"walking",
"walked",
"run",
"cat",
"walk",
"unknownzzz",
]
assert from_compiled.stem_batch(words) == from_text.stem_batch(words)
# The compiled artifact uses the gzip-wrapped EGTR v7 stream format.
import gzip
with gzip.open(out, "rb") as fh:
assert fh.read(4) == b"EGTR"
os.unlink(out)
def test_cache_does_not_change_results(tmp_path: Path):
path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path)
plain = Stemmer(path=path, backward=True, cache_size=0)
cached = Stemmer(path=path, backward=True, cache_size=1000)
words = ["running", "runs", "cats", "nope", "run", "running", "cats"]
assert cached.stem_batch(words) == plain.stem_batch(words)
# Repeated lookups exercise the cache-hit path.
assert cached.stem_batch(["running"] * 5) == ["run"] * 5
def test_default_cache_is_shared_across_scalar_and_batch_apis(tmp_path: Path):
root = "cacheable-root-value"
variant = "cacheable-root-values"
path = _write_gz_dict([f"{root}\t{variant}"], tmp_path)
cached = Stemmer(path=path, backward=True)
first = cached.stem(variant)
assert first == root
assert cached.stem(variant) is first
assert cached.stemWord(variant) is first
assert cached.stem_batch([variant])[0] is first
assert cached.stemWords([variant])[0] is first
disabled = Stemmer(path=path, backward=True, cache_size=0)
uncached_first = disabled.stem(variant)
uncached_second = disabled.stem(variant)
assert uncached_first == uncached_second == root
assert uncached_first is not uncached_second
def test_full_cache_keeps_existing_entries_without_admitting_new_ones(tmp_path: Path):
roots = ("first-cacheable-root", "second-cacheable-root")
variants = tuple(f"{root}-value" for root in roots)
path = _write_gz_dict(
[f"{root}\t{variant}" for root, variant in zip(roots, variants)], tmp_path
)
stemmer = Stemmer(path=path, backward=True, cache_size=1)
first = stemmer.stem(variants[0])
assert stemmer.stem(variants[0]) is first
uncached = stemmer.stem(variants[1])
assert uncached == roots[1]
assert stemmer.stem(variants[1]) == uncached
assert stemmer.stem(variants[1]) is not uncached
assert stemmer.stem(variants[0]) is first
def test_lowercase_false_assumes_prelowered(tmp_path: Path):
path = _write_gz_dict(["cat\tcats"], tmp_path)
s = Stemmer(path=path, backward=True, lowercase=False)
assert s.stem("cats") == "cat" # already-lowercase input works
assert s.stem("CATS") is None # not lowercased -> no match
def test_forward_prefix_stemming(tmp_path: Path):
# Forward traversal handles prefix-oriented morphology (RTL languages).
path = _write_gz_dict(["kitab\talkitab\talkitabu"], tmp_path)
s = Stemmer(path=path, backward=False, store_original=True)
assert s.stem("alkitab") == "kitab"
assert s.stem("alkitabu") == "kitab"
assert s.stem("kitab") == "kitab"
def test_stem_all_returns_candidates(tmp_path: Path):
path = _write_gz_dict(["run\trunning", "runn\trunning"], tmp_path)
s = Stemmer(path=path, backward=True, store_original=True)
alls = s.stem_all("running")
# "running" maps to both "run" and "runn"; both must be reachable.
assert set(alls) >= {"run", "runn"}
# Installed standard-model smoke test.
def test_installed_english_compiled_model():
s = Stemmer("en")
assert s.stem_batch(["running", "walked", "cats"]) == ["run", "walk", "cat"]

179
python/tests/test_standard_models.py Normal file
View File

@@ -0,0 +1,179 @@
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
from __future__ import annotations
import gzip
import hashlib
import json
from contextlib import nullcontext
from importlib import resources
from pathlib import Path
import pytest
import radixor
EXPECTED_MODEL_IDS = {
"cs-cz-default",
"da-dk-default",
"de-de-default",
"es-es-default",
"fa-ir-default",
"fi-fi-default",
"fr-fr-default",
"he-il-default",
"hu-hu-default",
"it-it-default",
"nb-no-default",
"nl-nl-default",
"nn-no-default",
"pl-pl-unimorph",
"pt-pt-default",
"ru-ru-default",
"sv-se-default",
"uk-ua-default",
"us-uk-default",
"yi-default",
}
def _manifest() -> dict:
ref = resources.files("radixor_models_standard").joinpath("manifest.json")
return json.loads(ref.read_text(encoding="utf-8"))
def test_standard_manifest_model_set_versions_and_license():
manifest = _manifest()
assert manifest["catalog_version"] == "2026.1"
assert manifest["distribution_version"] == "0.0.0"
assert manifest["format"] == {"compression": "gzip", "magic": "EGTR", "version": 7}
assert {model["id"] for model in manifest["models"]} == EXPECTED_MODEL_IDS
assert "pl-pl-polimorf" not in {model["id"] for model in manifest["models"]}
assert {model["version"] for model in manifest["models"]} == {"1.0.0"}
assert {model["provenance"]["license"] for model in manifest["models"]} == {
"CC-BY-SA-3.0"
}
def test_standard_artifact_checksums_and_v7_headers():
root = resources.files("radixor_models_standard")
for model in _manifest()["models"]:
data = root.joinpath("models").joinpath(f"{model['id']}.rxc").read_bytes()
assert hashlib.sha256(data).hexdigest() == model["sha256"]
assert gzip.decompress(data)[:8] == b"EGTR\x00\x00\x00\x07"
def test_missing_standard_data_package_is_actionable(monkeypatch):
original_files = radixor.importlib.resources.files
def missing(package: str):
if package == radixor._STANDARD_PACKAGE:
raise ModuleNotFoundError(package)
return original_files(package)
monkeypatch.setattr(radixor.importlib.resources, "files", missing)
with pytest.raises(
ModuleNotFoundError, match="pip install radixor-models-standard"
):
radixor.Stemmer("en")
def test_missing_standard_model_is_actionable():
with pytest.raises(FileNotFoundError, match="not in the standard Radixor catalog"):
radixor.Stemmer("zz-zz-default")
@pytest.mark.parametrize(
("mutation", "message"),
[
(
lambda manifest: manifest.update(catalog_version="2027.1"),
"incompatible or corrupt",
),
(
lambda manifest: manifest.update(format={"magic": "bad"}),
"incompatible or corrupt",
),
(
lambda manifest: manifest.update(distribution_version="2.0.0"),
"incompatible or corrupt",
),
],
)
def test_incompatible_manifest_is_actionable(
tmp_path: Path, monkeypatch, mutation, message
):
manifest = _manifest()
mutation(manifest)
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
with pytest.raises(RuntimeError, match=message):
radixor.Stemmer("en")
def test_checksum_failure_is_detected(tmp_path: Path, monkeypatch):
manifest = _manifest()
(tmp_path / "models").mkdir()
(tmp_path / "models" / "us-uk-default.rxc").write_bytes(b"not the model")
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
with pytest.raises(RuntimeError, match="SHA-256 validation"):
radixor.Stemmer("en")
@pytest.mark.parametrize(
("stream_header", "message"),
[
(b"NOPE\x00\x00\x00\x07", "EGTR format marker"),
(b"EGTR\x00\x00\x00\x08", "unsupported compiled format v8"),
],
)
def test_format_marker_and_version_are_validated(
tmp_path: Path, monkeypatch, stream_header: bytes, message: str
):
manifest = _manifest()
model = next(item for item in manifest["models"] if item["id"] == "us-uk-default")
data = gzip.compress(stream_header, mtime=0)
model["sha256"] = hashlib.sha256(data).hexdigest()
(tmp_path / "models").mkdir()
(tmp_path / "models" / "us-uk-default.rxc").write_bytes(data)
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
with pytest.raises(RuntimeError, match=message):
radixor.Stemmer("en")
def test_invalid_model_id_is_rejected_before_resource_lookup():
with pytest.raises(ValueError, match="Invalid Radixor model ID"):
radixor.Stemmer("../us-uk-default")