feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
This commit is contained in:
52
.github/RELEASING-PYTHON.md
vendored
Normal file
52
.github/RELEASING-PYTHON.md
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
# Releasing the Python distributions
|
||||
|
||||
This maintainer-only checklist is intentionally outside the public MkDocs site.
|
||||
|
||||
The repository descriptors use `0.0.0` as a non-release placeholder. Release
|
||||
workflows create isolated staging trees and inject the stable version selected
|
||||
by the tag. Never commit a release-number rewrite of the descriptors.
|
||||
Standard `.rxc` resources are also generated in that staging tree from the
|
||||
canonical model sources. The workflow compiles every model twice and rejects
|
||||
non-deterministic output; generated payload must never be added to Git.
|
||||
|
||||
## Validate without publishing
|
||||
|
||||
Run **Python Standard Models Release** manually with version `1.0.0`, then run
|
||||
**Python Native Release** with version `4.1.0`. `workflow_dispatch` validates
|
||||
artifacts but cannot publish. The native run must pass Linux x86-64, Linux
|
||||
ARM64, macOS universal2, and Windows x86-64.
|
||||
|
||||
For the Linux paths, maintainers can use `act` with rootless Podman and the
|
||||
event files under `.github/act/`. Do not pass production secrets to `act`.
|
||||
|
||||
## Publish in dependency order
|
||||
|
||||
Both tags must point to a commit already contained in `main`.
|
||||
|
||||
```bash
|
||||
git tag -a 'python-models-standard@1.0.0' \
|
||||
-m 'Python standard models 1.0.0'
|
||||
git push origin 'python-models-standard@1.0.0'
|
||||
```
|
||||
|
||||
Wait until the models workflow has published its GitHub Release and Pages
|
||||
index entry. Then publish the native distribution:
|
||||
|
||||
```bash
|
||||
git tag -a 'python@4.1.0' -m 'Python Radixor 4.1.0'
|
||||
git push origin 'python@4.1.0'
|
||||
```
|
||||
|
||||
Do not push both tags together: native publication requires the standard-model
|
||||
release to exist first.
|
||||
|
||||
## Artifact identity
|
||||
|
||||
Python releases use `SHA256SUMS` and GitHub keyless build-provenance
|
||||
attestations. They do not use the Java Maven OpenPGP key. Verify a downloaded
|
||||
artifact with:
|
||||
|
||||
```bash
|
||||
sha256sum --check SHA256SUMS
|
||||
gh attestation verify <artifact> --repo leogalambos/Radixor
|
||||
```
|
||||
5
.github/act/python-models-standard.json
vendored
Normal file
5
.github/act/python-models-standard.json
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"inputs": {
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
5
.github/act/python-native.json
vendored
Normal file
5
.github/act/python-native.json
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"inputs": {
|
||||
"version": "4.1.0"
|
||||
}
|
||||
}
|
||||
95
.github/workflows/pages.yml
vendored
95
.github/workflows/pages.yml
vendored
@@ -26,42 +26,43 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: pages-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
group: github-python-pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish-pages:
|
||||
name: Publish static reports
|
||||
build-pages:
|
||||
name: Build static reports
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Validate Gradle wrapper
|
||||
uses: gradle/actions/wrapper-validation@v4
|
||||
uses: gradle/actions/wrapper-validation@0b6dd653ba04f4f93bf581ec31e66cbd7dcb644d # v4
|
||||
|
||||
- name: Set up Temurin JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '21'
|
||||
|
||||
- name: Set up Gradle caching and instrumentation
|
||||
uses: gradle/actions/setup-gradle@v4
|
||||
uses: gradle/actions/setup-gradle@0b6dd653ba04f4f93bf581ec31e66cbd7dcb644d # v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
python-version: '3.14.6'
|
||||
|
||||
- name: Install MkDocs Material
|
||||
run: python -m pip install --upgrade pip mkdocs-material
|
||||
run: python -m pip install --disable-pip-version-check mkdocs-material==9.7.6
|
||||
|
||||
- name: Verify reproducibility inputs
|
||||
shell: bash
|
||||
@@ -326,18 +327,80 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdocs build --strict --config-file build/mkdocs/mkdocs.yml
|
||||
rsync -a --delete --exclude '.git' --exclude '.git/' --exclude 'builds/' build/mkdocs-site/ .gh-pages/
|
||||
rsync -a --delete --exclude '.git' --exclude '.git/' --exclude 'builds/' --exclude 'python/' build/mkdocs-site/ .gh-pages/
|
||||
mkdir -p .gh-pages/builds
|
||||
cp build/mkdocs-site/builds/index.html .gh-pages/builds/index.html
|
||||
cat > .gh-pages/.nojekyll <<EOF
|
||||
EOF
|
||||
|
||||
- name: Commit and push gh-pages
|
||||
- name: Prepare read-only publication candidate
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd .gh-pages
|
||||
rm -rf .gh-pages/python
|
||||
rm -f .gh-pages/.git
|
||||
if find .gh-pages -type l -print -quit | grep -q .; then
|
||||
echo 'Publication candidate contains a symbolic link.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload static-site candidate
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: pages-site-${{ github.run_id }}
|
||||
path: .gh-pages
|
||||
if-no-files-found: error
|
||||
include-hidden-files: true
|
||||
retention-days: 1
|
||||
|
||||
publish-pages:
|
||||
name: Publish static reports
|
||||
needs: build-pages
|
||||
runs-on: ubuntu-latest
|
||||
environment: python-github-pages
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Check out repository for publication
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: pages
|
||||
|
||||
- name: Select or initialize gh-pages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd pages
|
||||
if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then
|
||||
git fetch origin gh-pages:refs/remotes/origin/gh-pages
|
||||
git checkout -B gh-pages origin/gh-pages
|
||||
else
|
||||
git checkout --orphan gh-pages
|
||||
git rm -rf .
|
||||
fi
|
||||
|
||||
- name: Download static-site candidate
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: pages-site-${{ github.run_id }}
|
||||
path: candidate
|
||||
|
||||
- name: Validate and publish static site
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test ! -e candidate/python
|
||||
test ! -e candidate/.git
|
||||
if find candidate -type l -print -quit | grep -q .; then
|
||||
echo 'Publication candidate contains a symbolic link.' >&2
|
||||
exit 1
|
||||
fi
|
||||
rsync -a --delete \
|
||||
--exclude '.git' --exclude '.git/' \
|
||||
--exclude 'python' --exclude 'python/' candidate/ pages/
|
||||
cd pages
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
@@ -348,4 +411,4 @@ jobs:
|
||||
fi
|
||||
|
||||
git commit -m "Publish reports for run ${GITHUB_RUN_NUMBER}"
|
||||
git push origin gh-pages
|
||||
git push origin HEAD:gh-pages
|
||||
|
||||
235
.github/workflows/python-models-standard-release.yml
vendored
Normal file
235
.github/workflows/python-models-standard-release.yml
vendored
Normal file
@@ -0,0 +1,235 @@
|
||||
name: Python Standard Models Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'python-models-standard@*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Stable distribution version to validate without publishing
|
||||
required: true
|
||||
default: '1.0.0'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: github-python-pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build and verify standard models
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.release.outputs.version }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
|
||||
steps:
|
||||
- name: Check out repository
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c
|
||||
with:
|
||||
toolchain: 1.88.0
|
||||
|
||||
- name: Select and validate release
|
||||
id: release
|
||||
shell: bash
|
||||
env:
|
||||
REQUESTED_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
|
||||
tag="${GITHUB_REF_NAME}"
|
||||
else
|
||||
tag="python-models-standard@${REQUESTED_VERSION}"
|
||||
fi
|
||||
eval "$(./tools/parse-python-release-tag.sh "${tag}")"
|
||||
[[ "${PYTHON_DISTRIBUTION}" == 'radixor-models-standard' ]]
|
||||
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
|
||||
[[ "$(git rev-parse "${tag}^{commit}")" == "${GITHUB_SHA}" ]]
|
||||
git merge-base --is-ancestor "${GITHUB_SHA}" origin/main
|
||||
fi
|
||||
printf 'version=%s\ntag=%s\n' "${PYTHON_VERSION}" "${tag}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Install pinned build tools
|
||||
run: >-
|
||||
python -m pip install --disable-pip-version-check
|
||||
maturin==1.14.1 setuptools==80.9.0 wheel==0.45.1
|
||||
|
||||
- name: Compile models and build isolated release tree
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -rf build/python-release
|
||||
mkdir -p build/python-release/compiler-wheel build/python-release/compiler-runtime
|
||||
maturin build --release --locked --manifest-path python/Cargo.toml \
|
||||
--out build/python-release/compiler-wheel
|
||||
python -c "from pathlib import Path; import zipfile; wheels=list(Path('build/python-release/compiler-wheel').glob('*.whl')); assert len(wheels) == 1; zipfile.ZipFile(wheels[0]).extractall('build/python-release/compiler-runtime')"
|
||||
PYTHONPATH=build/python-release/compiler-runtime \
|
||||
python python/scripts/build_standard_models.py \
|
||||
--project build/python-release/models-standard \
|
||||
--distribution-version '${{ steps.release.outputs.version }}'
|
||||
python python/scripts/build_standard_distribution.py \
|
||||
--project build/python-release/models-standard \
|
||||
--outdir build/python-release/artifacts
|
||||
|
||||
- name: Verify archives and offline installation
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python python/scripts/verify_distributions.py \
|
||||
--standard-dir build/python-release/artifacts \
|
||||
--standard-version '${{ steps.release.outputs.version }}'
|
||||
python python/scripts/assemble_release.py \
|
||||
models-standard '${{ steps.release.outputs.version }}' \
|
||||
build/python-release/artifacts build/python-release/release
|
||||
python -m venv build/python-release/venv
|
||||
build/python-release/venv/bin/python -m pip install \
|
||||
--no-index --find-links build/python-release/release \
|
||||
radixor-models-standard
|
||||
build/python-release/venv/bin/python -c \
|
||||
"from importlib import resources; assert resources.files('radixor_models_standard').joinpath('manifest.json').is_file()"
|
||||
|
||||
- name: Prepare complete PEP 503 index candidate
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
index_root='build/python-release/index/python/simple'
|
||||
if [[ "${ACT:-false}" != 'true' ]] && git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then
|
||||
git fetch origin gh-pages:refs/remotes/origin/gh-pages
|
||||
git worktree add --detach build/python-release/pages origin/gh-pages
|
||||
mkdir -p "$(dirname "${index_root}")"
|
||||
if [[ -d build/python-release/pages/python/simple ]]; then
|
||||
cp -R build/python-release/pages/python/simple "${index_root}"
|
||||
fi
|
||||
fi
|
||||
python python/scripts/update_simple_index.py \
|
||||
--root "${index_root}" \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--package radixor-models-standard \
|
||||
--version '${{ steps.release.outputs.version }}' \
|
||||
--tag '${{ steps.release.outputs.tag }}' \
|
||||
--artifacts build/python-release/release
|
||||
|
||||
- name: Upload verified release candidate
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-models-standard-release-${{ steps.release.outputs.version }}
|
||||
path: build/python-release/release/*
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload package-index candidate
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-models-standard-index-${{ steps.release.outputs.version }}
|
||||
path: build/python-release/index/python/simple
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
publish:
|
||||
name: Publish immutable GitHub Release
|
||||
if: github.event_name == 'push'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
environment: python-github-release
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
attestations: write
|
||||
|
||||
steps:
|
||||
- name: Download verified release candidate
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-models-standard-release-${{ needs.build.outputs.version }}
|
||||
path: release
|
||||
|
||||
- name: Verify release inventory
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd release
|
||||
sha256sum --check SHA256SUMS
|
||||
awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files
|
||||
find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \
|
||||
-printf '%f\n' | LC_ALL=C sort > actual-files
|
||||
diff -u expected-files actual-files
|
||||
rm expected-files actual-files
|
||||
|
||||
- name: Attest package artifacts
|
||||
uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2
|
||||
with:
|
||||
subject-path: |
|
||||
release/*.whl
|
||||
release/*.tar.gz
|
||||
|
||||
- name: Create and publish draft release exactly once
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ needs.build.outputs.tag }}
|
||||
RELEASE_VERSION: ${{ needs.build.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then
|
||||
echo "Release already exists; refusing to replace its assets: ${RELEASE_TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
gh release create "${RELEASE_TAG}" \
|
||||
release/*.whl release/*.tar.gz release/SHA256SUMS \
|
||||
--verify-tag --draft --title "radixor-models-standard ${RELEASE_VERSION}" \
|
||||
--notes "Precompiled standard Radixor model distribution ${RELEASE_VERSION}."
|
||||
gh release edit "${RELEASE_TAG}" --draft=false
|
||||
|
||||
publish-index:
|
||||
name: Publish Python package index
|
||||
if: github.event_name == 'push'
|
||||
needs: [build, publish]
|
||||
runs-on: ubuntu-latest
|
||||
environment: python-github-pages
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Check out gh-pages only
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: gh-pages
|
||||
path: pages
|
||||
|
||||
- name: Download validated index candidate
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-models-standard-index-${{ needs.build.outputs.version }}
|
||||
path: candidate
|
||||
|
||||
- name: Commit package index
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p pages/python/simple
|
||||
rsync -a --delete candidate/ pages/python/simple/
|
||||
cd pages
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
git add python/simple
|
||||
git diff --cached --quiet && exit 0
|
||||
git commit -m 'Index radixor-models-standard ${{ needs.build.outputs.version }}'
|
||||
git push origin HEAD:gh-pages
|
||||
452
.github/workflows/python-release.yml
vendored
Normal file
452
.github/workflows/python-release.yml
vendored
Normal file
@@ -0,0 +1,452 @@
|
||||
name: Python Native Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'python@*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Stable distribution version to validate without publishing
|
||||
required: true
|
||||
default: '4.1.0'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: github-python-pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Prepare versioned sources and sdist
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.release.outputs.version }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
|
||||
steps:
|
||||
- name: Check out repository
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c
|
||||
with:
|
||||
toolchain: 1.88.0
|
||||
|
||||
- name: Select and validate release
|
||||
id: release
|
||||
shell: bash
|
||||
env:
|
||||
REQUESTED_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
|
||||
tag="${GITHUB_REF_NAME}"
|
||||
else
|
||||
tag="python@${REQUESTED_VERSION}"
|
||||
fi
|
||||
eval "$(./tools/parse-python-release-tag.sh "${tag}")"
|
||||
[[ "${PYTHON_DISTRIBUTION}" == 'radixor' ]]
|
||||
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
|
||||
[[ "$(git rev-parse "${tag}^{commit}")" == "${GITHUB_SHA}" ]]
|
||||
git merge-base --is-ancestor "${GITHUB_SHA}" origin/main
|
||||
fi
|
||||
printf 'version=%s\ntag=%s\n' "${PYTHON_VERSION}" "${tag}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Install pinned source-build tools
|
||||
run: >-
|
||||
python -m pip install --disable-pip-version-check
|
||||
maturin==1.14.1 setuptools==80.9.0 wheel==0.45.1
|
||||
|
||||
- name: Materialize versioned native source
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -rf build/python-release
|
||||
python python/scripts/prepare_release_tree.py \
|
||||
native '${{ steps.release.outputs.version }}' \
|
||||
build/python-release/native-source
|
||||
|
||||
- name: Build and verify source distributions
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p build/python-release/compiler-wheel build/python-release/compiler-runtime \
|
||||
build/python-release/native-sdist build/python-release/models
|
||||
maturin build --release --locked \
|
||||
--manifest-path build/python-release/native-source/Cargo.toml \
|
||||
--out build/python-release/compiler-wheel
|
||||
python -c "from pathlib import Path; import zipfile; wheels=list(Path('build/python-release/compiler-wheel').glob('*.whl')); assert len(wheels) == 1; zipfile.ZipFile(wheels[0]).extractall('build/python-release/compiler-runtime')"
|
||||
PYTHONPATH=build/python-release/compiler-runtime \
|
||||
python python/scripts/build_standard_models.py \
|
||||
--project build/python-release/models-source \
|
||||
--distribution-version 1.0.0
|
||||
maturin sdist --manifest-path build/python-release/native-source/Cargo.toml \
|
||||
--out build/python-release/native-sdist
|
||||
python python/scripts/build_standard_distribution.py \
|
||||
--project build/python-release/models-source \
|
||||
--outdir build/python-release/models
|
||||
python python/scripts/verify_distributions.py \
|
||||
--standard-dir build/python-release/models \
|
||||
--standard-version 1.0.0
|
||||
PYTHONPATH=python/scripts python -c \
|
||||
"from pathlib import Path; from verify_distributions import _verify_main_sdist; _verify_main_sdist(next(Path('build/python-release/native-sdist').glob('*.tar.gz')), '${{ steps.release.outputs.version }}')"
|
||||
|
||||
- name: Upload versioned native source
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: native-source-${{ steps.release.outputs.version }}
|
||||
path: |
|
||||
build/python-release/native-source
|
||||
!build/python-release/native-source/target/**
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload native sdist
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: native-sdist-${{ steps.release.outputs.version }}
|
||||
path: build/python-release/native-sdist/*.tar.gz
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload verified model fixture
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: native-model-fixture-${{ steps.release.outputs.version }}
|
||||
path: build/python-release/models/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
build-linux-x86-64:
|
||||
name: Build Linux x86-64 wheel
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c
|
||||
with:
|
||||
toolchain: 1.88.0
|
||||
|
||||
- name: Download versioned native source
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: native-source-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/native-source
|
||||
|
||||
- name: Download model fixture
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: native-model-fixture-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/models
|
||||
|
||||
- name: Select Linux build isolation
|
||||
id: linux-isolation
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${ACT:-}" == 'true' ]]; then
|
||||
echo 'manylinux=off' >> "${GITHUB_OUTPUT}"
|
||||
echo 'container=' >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo 'manylinux=auto' >> "${GITHUB_OUTPUT}"
|
||||
echo 'container=quay.io/pypa/manylinux2014_x86_64@sha256:0a42cb7e5f4ba6bbfb8d0a86d1aab0c8876ba9c3be16bd99360ae42bf010ec77' >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Build manylinux wheel
|
||||
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
|
||||
with:
|
||||
command: build
|
||||
target: x86_64
|
||||
manylinux: ${{ steps.linux-isolation.outputs.manylinux }}
|
||||
container: ${{ steps.linux-isolation.outputs.container }}
|
||||
maturin-version: v1.14.1
|
||||
rust-toolchain: 1.88.0
|
||||
working-directory: build/python-release/native-source
|
||||
args: --release --locked --out ../wheel
|
||||
|
||||
- name: Smoke-test wheel with standard models
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python -m pip install --no-index \
|
||||
--find-links build/python-release/wheel \
|
||||
--find-links build/python-release/models radixor
|
||||
python -c "from radixor import Stemmer; assert Stemmer('en').stem('running') == 'run'"
|
||||
|
||||
- name: Upload Linux x86-64 wheel
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: native-wheel-linux-x86-64-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/wheel/*.whl
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
build-platform-wheels:
|
||||
name: Build ${{ matrix.name }} wheel
|
||||
needs: prepare
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: Linux aarch64
|
||||
os: ubuntu-24.04-arm
|
||||
target: aarch64
|
||||
manylinux: auto
|
||||
container: quay.io/pypa/manylinux2014_aarch64@sha256:63bfa74be47f0277e998cb7c1b571b27664ac848bb356b0f4588438f930285dd
|
||||
artifact: linux-aarch64
|
||||
- name: macOS universal2
|
||||
os: macos-14
|
||||
target: universal2-apple-darwin
|
||||
manylinux: 'off'
|
||||
container: ''
|
||||
artifact: macos-universal2
|
||||
- name: Windows x86-64
|
||||
os: windows-2022
|
||||
target: x86_64-pc-windows-msvc
|
||||
manylinux: 'off'
|
||||
container: ''
|
||||
artifact: windows-x86-64
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c
|
||||
with:
|
||||
toolchain: 1.88.0
|
||||
|
||||
- name: Download versioned native source
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: native-source-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/native-source
|
||||
|
||||
- name: Download model fixture
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: native-model-fixture-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/models
|
||||
|
||||
- name: Build platform wheel
|
||||
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
|
||||
with:
|
||||
command: build
|
||||
target: ${{ matrix.target }}
|
||||
manylinux: ${{ matrix.manylinux }}
|
||||
container: ${{ matrix.container }}
|
||||
maturin-version: v1.14.1
|
||||
rust-toolchain: 1.88.0
|
||||
working-directory: build/python-release/native-source
|
||||
args: --release --locked --out ../wheel
|
||||
|
||||
- name: Smoke-test wheel with standard models
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python -m pip install --no-index \
|
||||
--find-links build/python-release/wheel \
|
||||
--find-links build/python-release/models radixor
|
||||
python -c "from radixor import Stemmer; assert Stemmer('en').stem('running') == 'run'"
|
||||
|
||||
- name: Upload platform wheel
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: native-wheel-${{ matrix.artifact }}-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/wheel/*.whl
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
assemble:
|
||||
name: Assemble verified release
|
||||
needs: [prepare, build-linux-x86-64, build-platform-wheels]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out repository
|
||||
if: ${{ env.ACT != 'true' }}
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: '3.12.13'
|
||||
|
||||
- name: Download native artifacts only
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
pattern: native-wheel-*
|
||||
path: build/python-release/artifacts
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download native sdist
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: native-sdist-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/artifacts
|
||||
|
||||
- name: Enforce release allowlist and checksums
|
||||
run: >-
|
||||
python python/scripts/assemble_release.py native
|
||||
'${{ needs.prepare.outputs.version }}'
|
||||
build/python-release/artifacts build/python-release/release
|
||||
|
||||
- name: Prepare complete PEP 503 index candidate
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
index_root='build/python-release/index/python/simple'
|
||||
if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then
|
||||
git fetch origin gh-pages:refs/remotes/origin/gh-pages
|
||||
git worktree add --detach build/python-release/pages origin/gh-pages
|
||||
mkdir -p "$(dirname "${index_root}")"
|
||||
if [[ -d build/python-release/pages/python/simple ]]; then
|
||||
cp -R build/python-release/pages/python/simple "${index_root}"
|
||||
fi
|
||||
fi
|
||||
python python/scripts/update_simple_index.py \
|
||||
--root "${index_root}" \
|
||||
--repository "${GITHUB_REPOSITORY}" \
|
||||
--package radixor \
|
||||
--version '${{ needs.prepare.outputs.version }}' \
|
||||
--tag '${{ needs.prepare.outputs.tag }}' \
|
||||
--artifacts build/python-release/release
|
||||
|
||||
- name: Upload verified release candidate
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-native-release-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/release/*
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload package-index candidate
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-native-index-${{ needs.prepare.outputs.version }}
|
||||
path: build/python-release/index/python/simple
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
publish:
|
||||
name: Publish immutable GitHub Release
|
||||
if: github.event_name == 'push'
|
||||
needs: [prepare, assemble]
|
||||
runs-on: ubuntu-latest
|
||||
environment: python-github-release
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
attestations: write
|
||||
|
||||
steps:
|
||||
- name: Download verified release candidate
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-native-release-${{ needs.prepare.outputs.version }}
|
||||
path: release
|
||||
|
||||
- name: Verify release inventory
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd release
|
||||
sha256sum --check SHA256SUMS
|
||||
awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files
|
||||
find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \
|
||||
-printf '%f\n' | LC_ALL=C sort > actual-files
|
||||
diff -u expected-files actual-files
|
||||
rm expected-files actual-files
|
||||
|
||||
- name: Require published standard models
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$(gh release view 'python-models-standard@1.0.0' --json isDraft --jq '.isDraft')" == 'false' ]]
|
||||
|
||||
- name: Attest package artifacts
|
||||
uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2
|
||||
with:
|
||||
subject-path: |
|
||||
release/*.whl
|
||||
release/*.tar.gz
|
||||
|
||||
- name: Create and publish draft release exactly once
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
RELEASE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then
|
||||
echo "Release already exists; refusing to replace its assets: ${RELEASE_TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
gh release create "${RELEASE_TAG}" \
|
||||
release/*.whl release/*.tar.gz release/SHA256SUMS \
|
||||
--verify-tag --draft --title "radixor ${RELEASE_VERSION}" \
|
||||
--notes "Native Rust/Python Radixor distribution ${RELEASE_VERSION}."
|
||||
gh release edit "${RELEASE_TAG}" --draft=false
|
||||
|
||||
publish-index:
|
||||
name: Publish Python package index
|
||||
if: github.event_name == 'push'
|
||||
needs: [prepare, assemble, publish]
|
||||
runs-on: ubuntu-latest
|
||||
environment: python-github-pages
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Check out gh-pages only
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: gh-pages
|
||||
path: pages
|
||||
|
||||
- name: Download validated index candidate
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-native-index-${{ needs.prepare.outputs.version }}
|
||||
path: candidate
|
||||
|
||||
- name: Commit package index
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p pages/python/simple
|
||||
rsync -a --delete candidate/ pages/python/simple/
|
||||
cd pages
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
git add python/simple
|
||||
git diff --cached --quiet && exit 0
|
||||
git commit -m 'Index radixor ${{ needs.prepare.outputs.version }}'
|
||||
git push origin HEAD:gh-pages
|
||||
21
.gitignore
vendored
21
.gitignore
vendored
@@ -88,6 +88,9 @@ local.properties
|
||||
# Typically, this file would be tracked if it contains build/dependency configurations:
|
||||
#.project
|
||||
|
||||
# Eclipse and Buildship create project descriptors during import.
|
||||
**/.project
|
||||
|
||||
# PMD plugin conf
|
||||
.pmd
|
||||
|
||||
@@ -109,3 +112,21 @@ gradle-app.setting
|
||||
|
||||
# Gradle task-name cache
|
||||
.gradletasknamecache
|
||||
|
||||
##---------------------------------------------------------------------------------------- Python tooling
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.mypy_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
htmlcov/
|
||||
|
||||
# Workspace-local Rust/Python build and test scratch directories.
|
||||
/.cargo-target/
|
||||
/pytest-of-*/
|
||||
|
||||
# tempfile.NamedTemporaryFile-style dictionary scratch files must not survive
|
||||
# as repository candidates when a process is interrupted.
|
||||
/tmp*.gz
|
||||
|
||||
22
.project
22
.project
@@ -1,22 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>Radixor</name>
|
||||
<comment></comment>
|
||||
<projects/>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
||||
</natures>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<linkedResources/>
|
||||
<filteredResources/>
|
||||
</projectDescription>
|
||||
105
README.md
105
README.md
@@ -1,16 +1,19 @@
|
||||
<img src="docs/assets/images/banner.jpg" width="100%" alt="Radixor banner" />
|
||||
<p align="center">
|
||||
<img src="docs/assets/images/radixor-logo.png" width="160" alt="Radixor logo" />
|
||||
</p>
|
||||
|
||||
[](LICENSE)
|
||||
[](#)
|
||||
[](docs/python/fast-track.md)
|
||||
[](https://central.sonatype.com/artifact/org.egothor/radixor)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/)
|
||||
[](https://github.com/leogalambos/Radixor/actions/workflows/build.yml)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||
|
||||
*Deterministic, multi-language stemming for Java, built around compact dictionary-derived patch-command tries with an explicit quality/speed trade-off.*
|
||||
*Deterministic, multi-language stemming for Java and Python, built around compact dictionary-trained patch-command tries with an explicit quality/speed trade-off.*
|
||||
|
||||
**Radixor** is a modern multi-language stemming toolkit for Java in the tradition of the original **Egothor** approach. It learns compact word-to-stem transformations from dictionary data, stores them in compiled patch-command tries, and exposes a runtime model designed for speed, determinism, and operational simplicity. Unlike a closed-form dictionary lookup stemmer, Radixor can also generalize beyond explicitly listed word forms.
|
||||
**Radixor** is a modern multi-language stemming toolkit for Java and Python in the tradition of the original **Egothor** approach. It learns compact word-to-stem transformations from dictionary data, stores them in compiled patch-command tries, and exposes native runtime implementations designed for speed, determinism, and operational simplicity. Unlike a closed-form dictionary lookup stemmer, Radixor can also generalize beyond explicitly listed word forms.
|
||||
|
||||
It is particularly well suited to systems that need stemming which is:
|
||||
|
||||
@@ -22,7 +25,35 @@ It is particularly well suited to systems that need stemming which is:
|
||||
|
||||
It also retains the operational advantages of a compiled artifact model: predictable runtime behavior, direct binary loading, and clear separation between preparation-time compilation and live request processing.
|
||||
|
||||
## Add Radixor and a model
|
||||
## Choose a runtime
|
||||
|
||||
For Python, one installation provides the native runtime and the separate
|
||||
standard package of 20 precompiled models:
|
||||
|
||||
From PyPI, once publication is enabled:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: radixor
|
||||
```
|
||||
|
||||
Or from the GitHub Releases-backed index:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: \
|
||||
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
|
||||
```
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
english = Stemmer("en")
|
||||
print(english.stemWord("running")) # run
|
||||
```
|
||||
|
||||
Continue with the [Python Fast Track](docs/python/fast-track.md) or
|
||||
[Python Quick Start](docs/python/quick-start.md).
|
||||
|
||||
### Java dependencies
|
||||
|
||||
The core artifact contains the algorithm and registry, but no language dictionary. Add either one minimal model or the optional standard default pack:
|
||||
|
||||
@@ -87,16 +118,16 @@ Radixor performance is best read together with stemming quality. The English dic
|
||||
|
||||
| Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token |
|
||||
| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 20.627 | 2.117 | 98.0 |
|
||||
| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.713 | 2.104 | 103.2 |
|
||||
| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 17.408 | 1.438 | 82.7 |
|
||||
| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 16.946 | 1.531 | 80.5 |
|
||||
| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 15.735 | 1.278 | 74.8 |
|
||||
| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 14.714 | 1.089 | 69.9 |
|
||||
| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.090 | 1.254 | 71.7 |
|
||||
| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 13.773 | 1.071 | 65.4 |
|
||||
| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 15.396 | 2.497 | 73.1 |
|
||||
| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 16.970 | 2.847 | 80.6 |
|
||||
| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 15.064 | 0.658 | 71.6 |
|
||||
| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 17.798 | 2.161 | 84.6 |
|
||||
| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 13.900 | 0.941 | 66.0 |
|
||||
| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 14.809 | 1.376 | 70.3 |
|
||||
| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 13.186 | 0.930 | 62.6 |
|
||||
| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 12.852 | 0.943 | 61.1 |
|
||||
| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 12.358 | 0.831 | 58.7 |
|
||||
| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 11.657 | 0.921 | 55.4 |
|
||||
| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 11.494 | 1.256 | 54.6 |
|
||||
| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 9.895 | 0.925 | 47.0 |
|
||||
|
||||
Column meanings:
|
||||
|
||||
@@ -109,7 +140,7 @@ Column meanings:
|
||||
- `Error ms` is the JMH score error converted to milliseconds.
|
||||
- `ns/token` is average nanoseconds per changed token in that operation.
|
||||
|
||||
The contracted trie result is materially stronger than the older uncontracted profile: full English coverage reaches 97.478% all-token exactness and 97.197% changed-token exactness at 98.0 ns/token, while even a 10% deterministic dictionary slice remains at 92.868% all-token exactness and 76.516% changed-token exactness at 80.6 ns/token. This is why Radixor benchmark results are documented with both speed and quality instead of a single Porter speed badge.
|
||||
The contracted trie result is materially stronger than the older uncontracted profile: full English coverage reaches 97.478% all-token exactness and 97.197% changed-token exactness at 71.6 ns/token, while even a 10% deterministic dictionary slice remains at 92.868% all-token exactness and 76.516% changed-token exactness at 47.0 ns/token. This is why Radixor benchmark results are documented with both speed and quality instead of a single Porter speed badge.
|
||||
|
||||
For benchmark scope, workload design, environment, commands, report locations, and interpretation guidance, see [Benchmarking](docs/benchmarking.md).
|
||||
|
||||
@@ -182,13 +213,22 @@ The repository keeps the front page concise and places detailed documentation un
|
||||
|
||||
### Getting Started
|
||||
|
||||
- [Fast Track](docs/fast-track.md)
|
||||
The shortest path from adding core plus a model artifact to getting a first stem.
|
||||
- [Python Fast Track](docs/python/fast-track.md)
|
||||
The shortest path from `pip install` to the first native Python stem.
|
||||
|
||||
- [Quick Start](docs/quick-start.md)
|
||||
A broader developer walkthrough covering loading options, querying, extension, persistence, and metadata.
|
||||
- [Java Fast Track](docs/fast-track.md)
|
||||
The shortest Java path from adding core plus a model artifact to getting a first stem.
|
||||
|
||||
- [Integration Deep Dive](docs/integration-deep-dive.md)
|
||||
- [Python Quick Start](docs/python/quick-start.md)
|
||||
Installation, standard models, batch use, PyStemmer migration, and deployment guidance.
|
||||
|
||||
- [Java Quick Start](docs/quick-start.md)
|
||||
A broader Java walkthrough covering loading options, querying, extension, persistence, and metadata.
|
||||
|
||||
- [Python Overview](docs/python/index.md)
|
||||
Runtime architecture, model packaging, API capabilities, and Java interoperability.
|
||||
|
||||
- [Java Integration Deep Dive](docs/integration-deep-dive.md)
|
||||
Dependency setup, model selection, production lifecycle, search-pipeline guidance, and operational checklist.
|
||||
|
||||
- [Built-in Languages](docs/built-in-languages.md)
|
||||
@@ -197,10 +237,29 @@ The repository keeps the front page concise and places detailed documentation un
|
||||
- [Dictionary Format](docs/dictionary-format.md)
|
||||
How to write and normalize stemming dictionaries.
|
||||
|
||||
- [Compilation (CLI tool)](docs/cli-compilation.md)
|
||||
How to compile dictionaries into deployable binary artifacts.
|
||||
- [Java CLI Compilation](docs/cli-compilation.md)
|
||||
How to compile dictionaries into deployable binary artifacts from Java.
|
||||
|
||||
### Programmatic Usage
|
||||
### Python
|
||||
|
||||
The Python installation installs the native package together with the pure
|
||||
`radixor-models-standard` 1.x distribution of the 2026.1 catalog: 20 precompiled v7 models, excluding
|
||||
the optional PoliMorf model. Python runtime distributions contain no textual
|
||||
dictionaries.
|
||||
|
||||
- [Installation and Builds](docs/python/installation.md)
|
||||
Wheels, source builds, Gradle tasks, host builds, and cross-compilation requirements.
|
||||
|
||||
- [Usage and API](docs/python/usage.md)
|
||||
Single and batch stemming, caching, custom dictionaries, and compiled models.
|
||||
|
||||
- [Dictionary Compilation](docs/python/model-compilation.md)
|
||||
Compile a textual dictionary once, load it directly, or share its version 7 binary with Java.
|
||||
|
||||
- [Python Benchmarks](docs/python/performance.md)
|
||||
Batch methodology and comparisons with available Python stemmers.
|
||||
|
||||
### Java Programmatic Usage
|
||||
|
||||
- [Programmatic Usage Overview](docs/programmatic-usage.md)
|
||||
Entry point to the Java API and the overall usage model.
|
||||
|
||||
BIN
Radixor.png
BIN
Radixor.png
Binary file not shown.
|
Before Width: | Height: | Size: 318 KiB |
@@ -762,7 +762,9 @@ tasks.register('prepareMkDocsSource', Sync) {
|
||||
buildsPage.setText('# Historical Builds\n\nThe Pages publication workflow replaces this staging placeholder with the retained build index.\n', 'UTF-8')
|
||||
File configuration = layout.buildDirectory.file('mkdocs/mkdocs.yml').get().asFile
|
||||
configuration.parentFile.mkdirs()
|
||||
configuration.setText(layout.projectDirectory.file('mkdocs.yml').asFile.getText('UTF-8')
|
||||
String configurationText = layout.projectDirectory.file('mkdocs.yml').asFile.getText('UTF-8')
|
||||
.replace('custom_dir: docs/overrides', 'custom_dir: ../mkdocs-source/overrides')
|
||||
configuration.setText(configurationText
|
||||
+ '\ndocs_dir: ../mkdocs-source\nsite_dir: ../mkdocs-site\n', 'UTF-8')
|
||||
}
|
||||
}
|
||||
@@ -1133,6 +1135,7 @@ apply from: 'gradle/paicehusk-benchmarks.gradle'
|
||||
apply from: 'gradle/opennlp-benchmarks.gradle'
|
||||
apply from: 'gradle/hunspell-benchmarks.gradle'
|
||||
apply from: 'gradle/cistem-benchmarks.gradle'
|
||||
apply from: 'gradle/python.gradle'
|
||||
|
||||
gradle.taskGraph.whenReady { taskGraph ->
|
||||
def banner = """
|
||||
|
||||
@@ -13,11 +13,29 @@ Radixor does not keep a large flat table of final stems. Instead, it converts di
|
||||
|
||||
The build-time flow is:
|
||||
|
||||
```text
|
||||
Dictionary -> Mutable trie -> Reduced trie -> Compiled trie
|
||||
```mermaid
|
||||
flowchart TD
|
||||
dictionary[Training dictionary]
|
||||
mutable[Mutable trie]
|
||||
reduced[Reduced trie]
|
||||
compiled[Compiled trie]
|
||||
|
||||
dictionary --> mutable --> reduced --> compiled
|
||||
```
|
||||
|
||||
For registered models, the dictionary is an independently versioned GZip resource discovered through a descriptor and verified before this flow begins. The model resource is input to trie construction, not a precompiled trie. See [Model Selection and Loading](model-selection-and-loading.md) for discovery and [Architecture](architecture.md) for component and release boundaries.
|
||||
Both implementations follow this conceptual flow. Java materializes its
|
||||
object-based compiled trie and exposes multiple reduction modes; the Python
|
||||
extension implements the production dominant-result profile in Rust and stores
|
||||
the runtime trie in flat arrays. Their persisted interoperability boundary is
|
||||
the version 7 binary stream, not their in-memory representation.
|
||||
|
||||
For registered Java models, the dictionary is an independently versioned GZip
|
||||
resource discovered through a descriptor and verified before this flow begins.
|
||||
For Python's standard models, this flow runs during package preparation and the
|
||||
installed `radixor-models-standard` distribution already contains validated
|
||||
compiled version 7 tries. See [Model Selection and Loading](model-selection-and-loading.md)
|
||||
for Java discovery and [Architecture](architecture.md) for component and release
|
||||
boundaries.
|
||||
|
||||
Explicit descriptors and stable model IDs now use the same compiled-value path as language defaults. `loadCompiled(descriptor, ...)` and `loadCompiled(modelId, ...)` first build with serialized patch commands and then map those values to `CompiledPatchCommand` while preserving metadata, reduction semantics, and ranked `getAll` order. Very large inputs can have a high temporary construction peak; PoliMorf is verified in an isolated 6 GiB JVM rather than increasing ordinary test or Gradle daemon heaps.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This document explains the structural architecture of **Radixor**: what data is stored, how it flows through the build pipeline, and how runtime lookup works once a compiled trie has been produced.
|
||||
|
||||
## Component boundaries
|
||||
## Java component boundaries
|
||||
|
||||
| Component | Responsibility |
|
||||
|---|---|
|
||||
@@ -19,6 +19,23 @@ This document explains the structural architecture of **Radixor**: what data is
|
||||
|
||||
Read [Model Selection and Loading](model-selection-and-loading.md) for executable application examples and [Stemmer Models](stemmer-models.md) for artifact maintenance.
|
||||
|
||||
## Python component boundaries
|
||||
|
||||
The Python distribution is a separate native implementation rather than a JVM
|
||||
wrapper. The `radixor` wheel contains the Rust/PyO3 runtime but no language
|
||||
data. Its mandatory `radixor-models-standard` dependency supplies 20 validated,
|
||||
precompiled, GZip-compressed version 7 `.rxc` tries. It does not use Java model
|
||||
JARs, `ServiceLoader`, descriptors, or the Java registry.
|
||||
|
||||
`Stemmer("<alias>")` resolves and synchronously loads a compiled standard model;
|
||||
it does not parse a textual dictionary at application startup.
|
||||
`radixor.compile(...)` remains available for application-owned textual
|
||||
dictionaries, and `Stemmer(compiled=...)` loads the resulting version 7
|
||||
artifact. The Java and Python in-memory layouts are intentionally different;
|
||||
the shared dictionary syntax and version 7 binary stream are their
|
||||
interoperability boundaries. See [Radixor for Python](python/index.md) and
|
||||
[Compiling Dictionaries in Python](python/model-compilation.md).
|
||||
|
||||
## Runtime model discovery and loading
|
||||
|
||||
The implemented sequence is:
|
||||
@@ -100,8 +117,14 @@ That matters because many words share similar transformation patterns. Once thos
|
||||
|
||||
The full build-time flow is:
|
||||
|
||||
```text
|
||||
Dictionary -> Mutable trie -> Reduced trie -> Compiled trie
|
||||
```mermaid
|
||||
flowchart TD
|
||||
dictionary[Training dictionary]
|
||||
mutable[Mutable trie]
|
||||
reduced[Reduced trie]
|
||||
compiled[Compiled trie]
|
||||
|
||||
dictionary --> mutable --> reduced --> compiled
|
||||
```
|
||||
|
||||
Each stage has a different purpose.
|
||||
|
||||
50
docs/assets/data/homepage-performance.json
Normal file
50
docs/assets/data/homepage-performance.json
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"source": "Python all-language batch benchmark, 2026-08-08",
|
||||
"environment": {
|
||||
"processor": "AMD Ryzen 5 5625U with Radeon Graphics",
|
||||
"platform": "Linux-7.1.6-201.fc44.x86_64-x86_64-with-glibc2.43",
|
||||
"python": "CPython 3.14.6",
|
||||
"cpu_governor": "schedutil"
|
||||
},
|
||||
"batch_size": 100,
|
||||
"direct_pystemmer_comparisons": 18,
|
||||
"direct_pystemmer_wins": 18,
|
||||
"geometric_mean_speedup_vs_pystemmer": 1.665046904842523,
|
||||
"maximum_speedup_vs_pystemmer": {
|
||||
"language": "it",
|
||||
"speedup": 3.027396185495421
|
||||
},
|
||||
"radixor_throughput_mwords_per_second": {
|
||||
"minimum": {
|
||||
"language": "ru",
|
||||
"value": 3.6574304356731138
|
||||
},
|
||||
"maximum": {
|
||||
"language": "pt",
|
||||
"value": 5.99085795076713
|
||||
}
|
||||
},
|
||||
"languages": {
|
||||
"cs": {"radixor_ns_per_word": 224.3, "radixor_mwords_per_second": 4.46, "pystemmer_ns_per_word": 236.6, "speedup_vs_pystemmer": 1.05},
|
||||
"da": {"radixor_ns_per_word": 178.3, "radixor_mwords_per_second": 5.61, "pystemmer_ns_per_word": 267.6, "speedup_vs_pystemmer": 1.50},
|
||||
"de": {"radixor_ns_per_word": 230.9, "radixor_mwords_per_second": 4.33, "pystemmer_ns_per_word": 635.5, "speedup_vs_pystemmer": 2.75},
|
||||
"en": {"radixor_ns_per_word": 180.5, "radixor_mwords_per_second": 5.54, "pystemmer_ns_per_word": 331.9, "speedup_vs_pystemmer": 1.84},
|
||||
"es": {"radixor_ns_per_word": 184.2, "radixor_mwords_per_second": 5.43, "pystemmer_ns_per_word": 316.6, "speedup_vs_pystemmer": 1.72},
|
||||
"fa": {"radixor_ns_per_word": 210.1, "radixor_mwords_per_second": 4.76, "pystemmer_ns_per_word": 497.1, "speedup_vs_pystemmer": 2.37},
|
||||
"fi": {"radixor_ns_per_word": 227.8, "radixor_mwords_per_second": 4.39, "pystemmer_ns_per_word": 258.8, "speedup_vs_pystemmer": 1.14},
|
||||
"fr": {"radixor_ns_per_word": 234.2, "radixor_mwords_per_second": 4.27, "pystemmer_ns_per_word": 503.7, "speedup_vs_pystemmer": 2.15},
|
||||
"he": {"radixor_ns_per_word": 228.6, "radixor_mwords_per_second": 4.37, "pystemmer_ns_per_word": null, "speedup_vs_pystemmer": null},
|
||||
"hu": {"radixor_ns_per_word": 198.2, "radixor_mwords_per_second": 5.04, "pystemmer_ns_per_word": 264.7, "speedup_vs_pystemmer": 1.34},
|
||||
"it": {"radixor_ns_per_word": 170.8, "radixor_mwords_per_second": 5.86, "pystemmer_ns_per_word": 517.0, "speedup_vs_pystemmer": 3.03},
|
||||
"nb": {"radixor_ns_per_word": 187.1, "radixor_mwords_per_second": 5.34, "pystemmer_ns_per_word": 239.7, "speedup_vs_pystemmer": 1.28},
|
||||
"nl": {"radixor_ns_per_word": 187.1, "radixor_mwords_per_second": 5.35, "pystemmer_ns_per_word": 354.8, "speedup_vs_pystemmer": 1.90},
|
||||
"nn": {"radixor_ns_per_word": 168.7, "radixor_mwords_per_second": 5.93, "pystemmer_ns_per_word": 231.2, "speedup_vs_pystemmer": 1.37},
|
||||
"pl": {"radixor_ns_per_word": 194.6, "radixor_mwords_per_second": 5.14, "pystemmer_ns_per_word": 214.5, "speedup_vs_pystemmer": 1.10},
|
||||
"pt": {"radixor_ns_per_word": 166.9, "radixor_mwords_per_second": 5.99, "pystemmer_ns_per_word": 293.2, "speedup_vs_pystemmer": 1.76},
|
||||
"ru": {"radixor_ns_per_word": 273.4, "radixor_mwords_per_second": 3.66, "pystemmer_ns_per_word": 414.4, "speedup_vs_pystemmer": 1.52},
|
||||
"sv": {"radixor_ns_per_word": 189.3, "radixor_mwords_per_second": 5.28, "pystemmer_ns_per_word": 212.5, "speedup_vs_pystemmer": 1.12},
|
||||
"uk": {"radixor_ns_per_word": 221.5, "radixor_mwords_per_second": 4.51, "pystemmer_ns_per_word": null, "speedup_vs_pystemmer": null},
|
||||
"yi": {"radixor_ns_per_word": 227.5, "radixor_mwords_per_second": 4.39, "pystemmer_ns_per_word": 624.2, "speedup_vs_pystemmer": 2.74}
|
||||
},
|
||||
"pystemmer_missing_for_radixor_languages": ["he", "uk"]
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 540 KiB |
1
docs/assets/images/flags/cs.svg
Normal file
1
docs/assets/images/flags/cs.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="32" fill="#fff"/><rect y="32" width="64" height="32" fill="#d7141a"/><path d="M0 0 34 32 0 64Z" fill="#11457e"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 381 B |
1
docs/assets/images/flags/da.svg
Normal file
1
docs/assets/images/flags/da.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#c8102e"/><rect x="19" width="8" height="64" fill="#fff"/><rect y="27" width="64" height="8" fill="#fff"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 385 B |
1
docs/assets/images/flags/de.svg
Normal file
1
docs/assets/images/flags/de.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="22" fill="#111"/><rect y="22" width="64" height="21" fill="#dd0000"/><rect y="43" width="64" height="21" fill="#ffce00"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/en.svg
Normal file
1
docs/assets/images/flags/en.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#234b9b"/><path d="M0 0 64 64M64 0 0 64" stroke="white" stroke-width="14"/><path d="M0 0 64 64M64 0 0 64" stroke="#d4253f" stroke-width="7"/><path d="M32 0v64M0 32h64" stroke="white" stroke-width="18"/><path d="M32 0v64M0 32h64" stroke="#d4253f" stroke-width="10"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 544 B |
1
docs/assets/images/flags/es.svg
Normal file
1
docs/assets/images/flags/es.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#aa151b"/><rect y="16" width="64" height="32" fill="#f1bf00"/><circle cx="23" cy="32" r="4" fill="#aa151b"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 387 B |
1
docs/assets/images/flags/fa.svg
Normal file
1
docs/assets/images/flags/fa.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="22" fill="#239f40"/><rect y="22" width="64" height="20" fill="#fff"/><rect y="42" width="64" height="22" fill="#da0000"/><circle cx="32" cy="32" r="5" fill="none" stroke="#da0000" stroke-width="2"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 467 B |
1
docs/assets/images/flags/fi.svg
Normal file
1
docs/assets/images/flags/fi.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#fff"/><rect x="19" width="9" height="64" fill="#003580"/><rect y="27" width="64" height="9" fill="#003580"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 388 B |
1
docs/assets/images/flags/fr.svg
Normal file
1
docs/assets/images/flags/fr.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="22" height="64" fill="#0055a4"/><rect x="22" width="20" height="64" fill="#fff"/><rect x="42" width="22" height="64" fill="#ef4135"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/he.svg
Normal file
1
docs/assets/images/flags/he.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#fff"/><rect y="13" width="64" height="6" fill="#0038b8"/><rect y="45" width="64" height="6" fill="#0038b8"/><path d="M32 22 39 35H25Zm0 20-7-13h14Z" fill="none" stroke="#0038b8" stroke-width="2.3"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 478 B |
1
docs/assets/images/flags/hu.svg
Normal file
1
docs/assets/images/flags/hu.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="22" fill="#ce2939"/><rect y="22" width="64" height="20" fill="#fff"/><rect y="42" width="64" height="22" fill="#477050"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/it.svg
Normal file
1
docs/assets/images/flags/it.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="22" height="64" fill="#009246"/><rect x="22" width="20" height="64" fill="#fff"/><rect x="42" width="22" height="64" fill="#ce2b37"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/nb.svg
Normal file
1
docs/assets/images/flags/nb.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#ba0c2f"/><rect x="18" width="12" height="64" fill="#fff"/><rect y="25" width="64" height="12" fill="#fff"/><rect x="21" width="6" height="64" fill="#00205b"/><rect y="28" width="64" height="6" fill="#00205b"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 489 B |
1
docs/assets/images/flags/nl.svg
Normal file
1
docs/assets/images/flags/nl.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="22" fill="#ae1c28"/><rect y="22" width="64" height="20" fill="#fff"/><rect y="42" width="64" height="22" fill="#21468b"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/nn.svg
Normal file
1
docs/assets/images/flags/nn.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#ba0c2f"/><rect x="18" width="12" height="64" fill="#fff"/><rect y="25" width="64" height="12" fill="#fff"/><rect x="21" width="6" height="64" fill="#00205b"/><rect y="28" width="64" height="6" fill="#00205b"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 489 B |
1
docs/assets/images/flags/pl.svg
Normal file
1
docs/assets/images/flags/pl.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="32" fill="#fff"/><rect y="32" width="64" height="32" fill="#dc143c"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 338 B |
1
docs/assets/images/flags/pt.svg
Normal file
1
docs/assets/images/flags/pt.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="25" height="64" fill="#046a38"/><rect x="25" width="39" height="64" fill="#da291c"/><circle cx="25" cy="32" r="7" fill="#ffcd00"/><circle cx="25" cy="32" r="4" fill="#fff"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 430 B |
1
docs/assets/images/flags/ru.svg
Normal file
1
docs/assets/images/flags/ru.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="22" fill="#fff"/><rect y="22" width="64" height="20" fill="#0039a6"/><rect y="42" width="64" height="22" fill="#d52b1e"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 390 B |
1
docs/assets/images/flags/sv.svg
Normal file
1
docs/assets/images/flags/sv.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#006aa7"/><rect x="19" width="8" height="64" fill="#fecc00"/><rect y="27" width="64" height="8" fill="#fecc00"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 391 B |
1
docs/assets/images/flags/uk.svg
Normal file
1
docs/assets/images/flags/uk.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="32" fill="#0057b7"/><rect y="32" width="64" height="32" fill="#ffd700"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 341 B |
1
docs/assets/images/flags/yi.svg
Normal file
1
docs/assets/images/flags/yi.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><clipPath id="c"><circle cx="32" cy="32" r="30"/></clipPath></defs><g clip-path="url(#c)"><rect width="64" height="64" fill="#fff"/><rect y="13" width="64" height="6" fill="#0038b8"/><rect y="45" width="64" height="6" fill="#0038b8"/><path d="M32 22 39 35H25Zm0 20-7-13h14Z" fill="none" stroke="#0038b8" stroke-width="2.3"/></g><circle cx="32" cy="32" r="30" fill="none" stroke="#d8e2ee" stroke-width="2"/></svg>
|
||||
|
After Width: | Height: | Size: 478 B |
1
docs/assets/images/hero-lines.svg
Normal file
1
docs/assets/images/hero-lines.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 700" fill="none"><defs><linearGradient id="g" x1="0" y1="0" x2="1" y2="1"><stop stop-color="#6759ff" stop-opacity=".10"/><stop offset=".5" stop-color="#2d8cff" stop-opacity=".13"/><stop offset="1" stop-color="#12bfa7" stop-opacity=".10"/></linearGradient></defs><g stroke="url(#g)" stroke-width="1.25"><path d="M-40 585C150 328 262 493 408 255S665 12 927 50"/><path d="M-55 620C142 363 274 507 424 273S689 16 940 78"/><path d="M-65 652C151 403 289 526 447 301S704 34 958 112"/><path d="M-71 679C162 442 311 548 471 336S727 63 970 148"/><path d="M-74 704C186 481 338 574 500 371S748 94 985 190"/><path d="M244 8C400 46 516 22 622 93S780 236 940 227"/><path d="M270 -7C428 28 540 8 655 78S811 209 964 197"/></g><g fill="#0ea5e9" opacity=".18"><circle cx="408" cy="255" r="3.8"/><circle cx="500" cy="371" r="3.8"/><circle cx="655" cy="78" r="3.8"/><circle cx="811" cy="209" r="3.8"/></g></svg>
|
||||
|
After Width: | Height: | Size: 947 B |
1
docs/assets/images/java-award.svg
Normal file
1
docs/assets/images/java-award.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 220 190" fill="none"><defs><linearGradient id="s" x1="35" y1="25" x2="173" y2="172"><stop stop-color="#8b78ff"/><stop offset=".55" stop-color="#5b49e9"/><stop offset="1" stop-color="#3158f5"/></linearGradient><filter id="f"><feDropShadow dx="0" dy="8" stdDeviation="8" flood-color="#4f46e5" flood-opacity=".18"/></filter></defs><g stroke="#7667f2" stroke-width="5" stroke-linecap="round" fill="none" opacity=".8"><path d="M49 143c-23-9-32-25-37-45M42 127c-12-3-21-11-26-21M171 143c23-9 32-25 37-45M178 127c12-3 21-11 26-21"/><path d="M51 144c-17 6-28 7-38 5M169 144c17 6 28 7 38 5"/></g><path filter="url(#f)" d="M110 18 168 42v52c0 41-25 66-58 82-33-16-58-41-58-82V42Z" fill="url(#s)" stroke="#b7afff" stroke-width="3"/><path d="M88 114h44M96 124h28" stroke="white" stroke-width="5" stroke-linecap="round"/><path d="M90 92c0 12 8 20 20 20s20-8 20-20v-6H90v6Z" fill="none" stroke="white" stroke-width="5"/><path d="M130 88h8c9 0 9 13 0 13h-9" stroke="white" stroke-width="5" stroke-linecap="round"/><path d="M102 79c-13-12 9-16-2-30M115 78c-10-11 8-15 0-27M126 79c-8-9 7-13 0-23" stroke="white" stroke-width="4" stroke-linecap="round"/></svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
BIN
docs/assets/images/radixor-logo.png
Normal file
BIN
docs/assets/images/radixor-logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 74 KiB |
1
docs/assets/javascripts/landing-v2.js
Normal file
1
docs/assets/javascripts/landing-v2.js
Normal file
@@ -0,0 +1 @@
|
||||
document.addEventListener("DOMContentLoaded",()=>{const b=document.querySelector(".rx2-menu"),n=document.querySelector(".rx2-nav");if(b&&n)b.addEventListener("click",()=>{const o=n.classList.toggle("open");b.setAttribute("aria-expanded",String(o));});document.querySelectorAll("[data-copy]").forEach(btn=>btn.addEventListener("click",async()=>{const el=document.getElementById(btn.dataset.copy);if(!el)return;try{await navigator.clipboard.writeText(el.innerText);const t=btn.textContent;btn.textContent="Copied";setTimeout(()=>btn.textContent=t,1200);}catch(e){}}));});
|
||||
14
docs/assets/javascripts/mermaid.js
Normal file
14
docs/assets/javascripts/mermaid.js
Normal file
@@ -0,0 +1,14 @@
|
||||
const renderRadixorDiagrams = () => {
|
||||
mermaid.initialize({
|
||||
startOnLoad: false,
|
||||
theme: "neutral",
|
||||
flowchart: { htmlLabels: true, useMaxWidth: true },
|
||||
});
|
||||
return mermaid.run({ querySelector: ".mermaid" });
|
||||
};
|
||||
|
||||
if (typeof document$ === "undefined") {
|
||||
document.addEventListener("DOMContentLoaded", renderRadixorDiagrams);
|
||||
} else {
|
||||
document$.subscribe(renderRadixorDiagrams);
|
||||
}
|
||||
136
docs/assets/stylesheets/landing-v2.css
Normal file
136
docs/assets/stylesheets/landing-v2.css
Normal file
File diff suppressed because one or more lines are too long
146
docs/assets/stylesheets/radixor-docs-safety.css
Normal file
146
docs/assets/stylesheets/radixor-docs-safety.css
Normal file
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Radixor documentation safety constraints.
|
||||
*
|
||||
* The landing page uses custom page chrome while standard documentation pages
|
||||
* use Material for MkDocs. These size constraints ensure that a documentation
|
||||
* logo can never inherit unconstrained intrinsic dimensions. The remaining
|
||||
* rules carry the landing page's light blue/teal visual language into the
|
||||
* standard Material documentation shell.
|
||||
*/
|
||||
:root,
|
||||
[data-md-color-scheme="default"] {
|
||||
--md-primary-fg-color: #ffffff;
|
||||
--md-primary-fg-color--light: #ffffff;
|
||||
--md-primary-fg-color--dark: #f7faff;
|
||||
--md-primary-bg-color: #11182e;
|
||||
--md-primary-bg-color--light: #4b5b74;
|
||||
--md-accent-fg-color: #1769ef;
|
||||
--md-accent-fg-color--transparent: rgba(23, 105, 239, 0.1);
|
||||
--md-typeset-a-color: #0e62df;
|
||||
--rx-doc-line: #dce6f2;
|
||||
--rx-doc-soft: #f7faff;
|
||||
--rx-doc-text: #172554;
|
||||
--rx-doc-muted: #53627b;
|
||||
--rx-doc-teal: #078d7a;
|
||||
--rx-doc-purple: #6544ef;
|
||||
}
|
||||
|
||||
.md-header {
|
||||
color: var(--rx-doc-text);
|
||||
background: rgba(255, 255, 255, 0.97);
|
||||
border-bottom: 1px solid var(--rx-doc-line);
|
||||
box-shadow: 0 5px 18px rgba(33, 64, 112, 0.06);
|
||||
backdrop-filter: blur(8px);
|
||||
}
|
||||
|
||||
.md-header__title,
|
||||
.md-header__button,
|
||||
.md-header__topic,
|
||||
.md-source {
|
||||
color: var(--rx-doc-text);
|
||||
}
|
||||
|
||||
.md-search__form {
|
||||
background: #f3f7fc;
|
||||
border: 1px solid #d8e4f0;
|
||||
}
|
||||
|
||||
.md-search__input,
|
||||
.md-search__icon {
|
||||
color: var(--rx-doc-text);
|
||||
}
|
||||
|
||||
.md-search__input::placeholder {
|
||||
color: #69778d;
|
||||
}
|
||||
|
||||
.md-tabs {
|
||||
color: var(--rx-doc-text);
|
||||
background: #fbfdff;
|
||||
border-bottom: 1px solid var(--rx-doc-line);
|
||||
}
|
||||
|
||||
.md-nav__link--active,
|
||||
.md-nav__link:focus,
|
||||
.md-nav__link:hover {
|
||||
color: #145ff5;
|
||||
}
|
||||
|
||||
.md-nav__item--section > .md-nav__link {
|
||||
color: #263653;
|
||||
}
|
||||
|
||||
.md-typeset h1,
|
||||
.md-typeset h2,
|
||||
.md-typeset h3 {
|
||||
color: #0b1537;
|
||||
}
|
||||
|
||||
.md-typeset a:hover {
|
||||
color: var(--rx-doc-purple);
|
||||
}
|
||||
|
||||
.md-typeset code {
|
||||
border-radius: 0.22rem;
|
||||
background: #f3f7fc;
|
||||
}
|
||||
|
||||
.md-typeset .admonition,
|
||||
.md-typeset details {
|
||||
border-color: #8bb6f2;
|
||||
box-shadow: 0 3px 12px rgba(33, 64, 112, 0.06);
|
||||
}
|
||||
|
||||
.md-typeset table:not([class]) {
|
||||
border-color: var(--rx-doc-line);
|
||||
box-shadow: 0 2px 10px rgba(33, 64, 112, 0.04);
|
||||
}
|
||||
|
||||
.md-footer {
|
||||
--md-footer-fg-color: var(--rx-doc-text);
|
||||
--md-footer-fg-color--light: #31516f;
|
||||
--md-footer-fg-color--lighter: var(--rx-doc-muted);
|
||||
--md-footer-bg-color: #fbfdff;
|
||||
--md-footer-bg-color--dark: #f5f9fe;
|
||||
color: var(--rx-doc-muted);
|
||||
background: #fbfdff;
|
||||
border-top: 1px solid var(--rx-doc-line);
|
||||
}
|
||||
|
||||
.md-footer-meta {
|
||||
color: var(--rx-doc-muted);
|
||||
background: #f5f9fe;
|
||||
}
|
||||
|
||||
.md-footer a,
|
||||
.md-footer-meta a {
|
||||
color: #31516f;
|
||||
}
|
||||
|
||||
.md-header__button.md-logo img,
|
||||
.md-header__button.md-logo svg,
|
||||
.md-nav__button.md-logo img,
|
||||
.md-nav__button.md-logo svg {
|
||||
width: 1.6rem;
|
||||
height: 1.6rem;
|
||||
max-width: 1.6rem;
|
||||
max-height: 1.6rem;
|
||||
object-fit: cover;
|
||||
border-radius: 0.38rem;
|
||||
box-shadow: 0 2px 8px rgba(24, 82, 176, 0.18);
|
||||
}
|
||||
|
||||
.md-typeset .mermaid {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.md-typeset .mermaid svg {
|
||||
display: block;
|
||||
width: auto;
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
margin-inline: auto;
|
||||
}
|
||||
@@ -35,8 +35,8 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S
|
||||
"ENGLISH_RADIXOR","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","ALL_CANDIDATES","374384","568441","228735","555084","13357","1355","584042","374506","311382","15","0","161561989623","15","161561989638","0.000000","0","311382","0.000000","0.999951829979","1.000000000000","0.999999999907","0.999999999907","0.999999999954","0.999961463612","0.999975914409","0.999990365625","0.999951829979","0.999975914699","0.999975914653","0.000000000093","","","","",""
|
||||
"ENGLISH_SNOWBALL_ORIGINAL_PORTER","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","321092","284940","360538","28415","175199063592","360538","175199424130","0.000206","28415","313355","9.067990","0.441440296958","0.909320100206","0.999997942128","0.999997779945","0.954659021167","0.492078968883","0.594347503684","0.750277266077","0.422826769235","0.633569676567","0.633568843266","0.000002220055","","","","",""
|
||||
"ENGLISH_SNOWBALL_ORIGINAL_PORTER","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","299877","283312","357325","28070","161561632313","357325","161561989638","0.000221","28070","311382","9.014651","0.442234838138","0.909853491852","0.999997788310","0.999997614573","0.954925640081","0.492899966248","0.595181398691","0.751026553880","0.423671353822","0.634325556555","0.634324660984","0.000002385427","","","","",""
|
||||
"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","318385","284971","371381","28384","175199052749","371381","175199424130","0.000212","28384","313355","9.058097","0.434174040759","0.909419029535","0.999997880238","0.999997718233","0.954708454887","0.484848557029","0.587746607996","0.746086443827","0.416176453407","0.628367833992","0.628366984426","0.000002281767","","","","",""
|
||||
"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","297220","283368","368027","28014","161561621611","368027","161561989638","0.000228","28014","311382","8.996666","0.435017155489","0.910033335260","0.999997722069","0.999997548679","0.955015528665","0.485724531207","0.588647215295","0.746914872138","0.417080138768","0.629190045142","0.629189132274","0.000002451321","","","","",""
|
||||
"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","318389","284986","371197","28369","175199052933","371197","175199424130","0.000212","28369","313355","9.053310","0.434308721805","0.909466898566","0.999997881289","0.999997719369","0.954732389927","0.484985638615","0.587880000578","0.746191747709","0.416310229172","0.628481826499","0.628480977278","0.000002280631","","","","",""
|
||||
"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","297224","283383","367843","27999","161561621795","367843","161561989638","0.000228","27999","311382","8.991849","0.435153080497","0.910081507602","0.999997723208","0.999997549911","0.955039615405","0.485862840613","0.588781726310","0.747020963182","0.417215208510","0.629304990872","0.629304078379","0.000002450089","","","","",""
|
||||
"FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","439975","12317229","1508153","19148370","1599840231184","1508153","1599841739337","0.000094","19148370","31465599","60.854936","0.890914189568","0.391450644242","0.999999057311","0.999987088650","0.695724850776","0.709786610775","0.543915310644","0.440884276934","0.373546480243","0.590549687554","0.590545009664","0.000012911350","","","","",""
|
||||
"FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","431848","11954192","1155011","18806691","1504704980042","1155011","1504706135053","0.000077","18806691","30760883","61.138333","0.911893118140","0.388616672675","0.999999232401","0.999986734091","0.694307952538","0.718420864905","0.544981470973","0.438999334093","0.374552942180","0.595295615141","0.595290947645","0.000013265909","","","","",""
|
||||
"FINNISH_RADIXOR","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","69091","30511413","804","954186","1599841738533","804","1599841739337","0.000000","954186","31465599","3.032474","0.999973649899","0.969675263452","0.999999999497","0.999999403084","0.984837631475","0.993763441201","0.984591422195","0.975587162604","0.969650487220","0.984707932542","0.984707638627","0.000000596916","","","","",""
|
||||
@@ -205,24 +205,26 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S
|
||||
"RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","37297","758584","10","758584","0","1","758584","37282","12780071","0","255156","287711428009","0","287711428009","0.000000","255156","13035227","1.957434","1.000000000000","0.980425657336","1.000000000000","0.999999113193","0.990212828668","0.996022851412","0.990116093179","0.984278980143","0.980425657336","0.990164459742","0.990164020680","0.000000886807","","","","",""
|
||||
"RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","ANY_CANDIDATE","37297","758584","10","749142","9442","4","768163","37306","","","","","0","287711428009","0.000000","0","13035227","0.000000","","","","","","","","","","","","","","","","",""
|
||||
"RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","ALL_CANDIDATES","37297","758584","10","749142","9442","4","768163","37306","13035227","0","0","287711428009","0","287711428009","0.000000","0","13035227","0.000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","0.000000000000","","","","",""
|
||||
"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5553","78545","4795","11150","389682670","4795","389687465","0.001230","11150","89695","12.431016","0.942464602832","0.875689837784","0.999987695268","0.999959092010","0.937838766526","0.928307194100","0.907851012801","0.888276938388","0.831251984337","0.908463909669","0.908443737019","0.000040907990","","","","",""
|
||||
"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5539","78440","4795","11100","388399540","4795","388404335","0.001235","11100","89540","12.396694","0.942392022587","0.876033057851","0.999987654618","0.999959085584","0.938010356234","0.928327968188","0.908001736362","0.888546539947","0.831504743732","0.908606936602","0.908586757624","0.000040914416","","","","",""
|
||||
"SNOWBALL_CZECH_DIRECT","CS_CZ","cs-cz-default","1.0.0","62afdaa6dc7a721b54a0dc278a0c648a63ad52a34a412d27b5b52fbcde9c1ce4","ALL_WORDS","PRIMARY_OUTPUT","5113","51401","2","51401","0","1","51401","10932","172114","11935","128395","1320693256","11935","1320705191","0.000904","128395","300509","42.725842","0.935153138566","0.572741581783","0.999990963161","0.999893770330","0.786366272472","0.830101137739","0.710395865923","0.620863799839","0.550863514742","0.731847721723","0.731803909891","0.000106229670","","","","",""
|
||||
"SNOWBALL_CZECH_DIRECT","CS_CZ","cs-cz-default","1.0.0","62afdaa6dc7a721b54a0dc278a0c648a63ad52a34a412d27b5b52fbcde9c1ce4","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","5038","50697","2","50697","0","1","50697","10817","169464","11863","128023","1284758206","11863","1284770069","0.000923","128023","297487","43.034822","0.934576759115","0.569651783103","0.999990766441","0.999891145022","0.784821274772","0.828435805807","0.707848976847","0.617906692677","0.547806691450","0.729646021901","0.729601212980","0.000108854978","","","","",""
|
||||
"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5409","79378","4816","10317","389682649","4816","389687465","0.001236","10317","89695","11.502313","0.942798774259","0.884976866046","0.999987641378","0.999961175252","0.942482253712","0.930637722143","0.912973218547","0.895966806178","0.839881072045","0.913430404878","0.913411201871","0.000038824748","","","","",""
|
||||
"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5403","79223","4816","10317","388399519","4816","388404335","0.001240","10317","89540","11.522225","0.942693273361","0.884777752960","0.999987600550","0.999961047005","0.942382676755","0.930511444787","0.912817794779","0.895784477125","0.839618042308","0.913276538697","0.913257272617","0.000038952995","","","","",""
|
||||
"SNOWBALL_DANISH_LUCENE_FILTER","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5546","78557","4961","11138","389682504","4961","389687465","0.001273","11138","89695","12.417638","0.940599631217","0.875823624505","0.999987269285","0.999958696913","0.937905446895","0.926889068757","0.907056629699","0.888055112164","0.829920977011","0.907633945058","0.907613558620","0.000041303087","","","","",""
|
||||
"SNOWBALL_DANISH_LUCENE_FILTER","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5539","78440","4795","11100","388399540","4795","388404335","0.001235","11100","89540","12.396694","0.942392022587","0.876033057851","0.999987654618","0.999959085584","0.938010356234","0.928327968188","0.908001736362","0.888546539947","0.831504743732","0.908606936602","0.908586757624","0.000040914416","","","","",""
|
||||
"SNOWBALL_DUTCH_DIRECT","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","ALL_WORDS","PRIMARY_OUTPUT","4992","26201","85","26201","0","1","26201","12051","29267","2987","35170","343165676","2987","343168663","0.000870","35170","64437","54.580443","0.907391331308","0.454195570868","0.999991295825","0.999888830652","0.727093433346","0.756436964017","0.605371751249","0.504599968276","0.434073920266","0.641975952605","0.641933549660","0.000111169348","","","","",""
|
||||
"SNOWBALL_DUTCH_DIRECT","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4796","25402","84","25402","0","1","25402","11466","29053","2987","33965","322552096","2987","322555083","0.000926","33965","63018","53.897299","0.906772784020","0.461027008156","0.999990739566","0.999885462099","0.730508873861","0.759841613575","0.611268909508","0.511294841471","0.440163623968","0.646565343716","0.646521311443","0.000114537901","","","","",""
|
||||
"SNOWBALL_DUTCH_LUCENE_FILTER","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","ALL_WORDS","PRIMARY_OUTPUT","4992","26201","85","26201","0","1","26201","14573","15204","759","49233","343167904","759","343168663","0.000221","49233","64437","76.404861","0.952452546514","0.235951394385","0.999997788260","0.999854349712","0.617974591322","0.592568341791","0.378208955224","0.277738198319","0.233204491073","0.474059602198","0.474021680915","0.000145650288","","","","",""
|
||||
"SNOWBALL_DUTCH_LUCENE_FILTER","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4796","25402","84","25402","0","1","25402","14116","14874","715","48144","322554368","715","322555083","0.000222","48144","63018","76.397220","0.954134325486","0.236027801581","0.999997783324","0.999848554685","0.618012792452","0.593185189912","0.378439579172","0.277851461363","0.233379881694","0.474554767395","0.474515425112","0.000151445315","","","","",""
|
||||
"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","381483","15082807","952306","16382792","1599840787031","952306","1599841739337","0.000060","16382792","31465599","52.065724","0.940611207417","0.479342757784","0.999999404750","0.999989164705","0.739671081267","0.788799811426","0.635056038739","0.531468350160","0.465261620083","0.671472389727","0.671468317739","0.000010835295","","","","",""
|
||||
"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","372232","14663371","936765","16097512","1504705198288","936765","1504706135053","0.000062","16097512","30760883","52.331112","0.939951485038","0.476688884386","0.999999377443","0.999988679564","0.738344130915","0.786987247415","0.632573283171","0.528815026735","0.462601231486","0.669376145960","0.669371899615","0.000011320436","","","","",""
|
||||
"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","381016","15095314","952479","16370285","1599840786858","952479","1599841739337","0.000060","16370285","31465599","52.025976","0.940647352567","0.479740239491","0.999999404642","0.999989172414","0.739869822067","0.789035310423","0.635413022080","0.531861528280","0.465644940456","0.671763638639","0.671759568086","0.000010827586","","","","",""
|
||||
"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","371779","14675605","936938","16085278","1504705198115","936938","1504706135053","0.000062","16085278","30760883","52.291340","0.939988123652","0.477086597287","0.999999377328","0.999988687580","0.738542987307","0.787224487482","0.632931670824","0.529208871663","0.462984663835","0.669668377186","0.669664132316","0.000011312420","","","","",""
|
||||
"SNOWBALL_FINNISH_LUCENE_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","377778","15121052","1288634","16344547","1599840450703","1288634","1599841739337","0.000081","16344547","31465599","51.944179","0.921471136011","0.480558212161","0.999999194524","0.999988978388","0.740278703342","0.778598131291","0.631685095974","0.531413183368","0.461651842069","0.665447610018","0.665443363449","0.000011021612","","","","",""
|
||||
"SNOWBALL_FINNISH_LUCENE_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","372232","14663371","936765","16097512","1504705198288","936765","1504706135053","0.000062","16097512","30760883","52.331112","0.939951485038","0.476688884386","0.999999377443","0.999988679564","0.738344130915","0.786987247415","0.632573283171","0.528815026735","0.462601231486","0.669376145960","0.669371899615","0.000011320436","","","","",""
|
||||
"SNOWBALL_FRENCH_DIRECT","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","ALL_WORDS","PRIMARY_OUTPUT","59240","404011","2301","404011","0","1","404011","85627","3744838","1092238","1625361","81605779618","1092238","81606871856","0.001338","1625361","5370199","30.266309","0.774194575401","0.697336914330","0.999986615858","0.999966701086","0.848661765094","0.757496924470","0.733758618240","0.711462917671","0.579477680015","0.734761496202","0.734744993787","0.000033298914","","","","",""
|
||||
"SNOWBALL_FRENCH_DIRECT","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","57698","400712","2133","400712","0","1","400712","84526","3736871","1088903","1619380","80278407962","1088903","80279496865","0.001356","1619380","5356251","30.233460","0.774356818202","0.697665400669","0.999986436101","0.999966266576","0.848825918385","0.757698693319","0.734013322497","0.711763857056","0.579795455624","0.735011537210","0.734994818860","0.000033733424","","","","",""
|
||||
"SNOWBALL_FRENCH_LUCENE_FILTER","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","ALL_WORDS","PRIMARY_OUTPUT","59240","404011","2301","404011","0","1","404011","85202","3742072","1097843","1628127","81605774013","1097843","81606871856","0.001345","1628127","5370199","30.317815","0.773168950281","0.696821849619","0.999986547175","0.999966598516","0.848404198397","0.756589837411","0.733012775372","0.710860736247","0.578547882033","0.734003418250","0.733986862867","0.000033401484","","","","",""
|
||||
"SNOWBALL_FRENCH_LUCENE_FILTER","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","57698","400712","2133","400712","0","1","400712","84810","3734232","1086494","1622019","80278410371","1086494","80279496865","0.001353","1622019","5356251","30.282729","0.774620254294","0.697172705312","0.999986466109","0.999966263711","0.848579585710","0.757784104203","0.733858787339","0.711398006457","0.579602638316","0.734876927298","0.734860210439","0.000033736289","","","","",""
|
||||
"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","81649","742376","65811","602476","38436668082","65811","38436733893","0.000171","602476","1344852","44.798684","0.918569588474","0.552013158325","0.999998287810","0.999982613933","0.776005723068","0.810879063265","0.689607573295","0.599890587538","0.526260347085","0.712083211201","0.712076031428","0.000017386067","","","","",""
|
||||
"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","37843","506459","41477","351958","10594922057","41477","10594963534","0.000391","351958","858417","41.000819","0.924303203294","0.589991810507","0.999996085215","0.999962868855","0.794993947861","0.830216831177","0.720244490537","0.635998708058","0.562798507380","0.738465517386","0.738449797528","0.000037131145","","","","",""
|
||||
"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","81641","742393","65811","602459","38436668082","65811","38436733893","0.000171","602459","1344852","44.797420","0.918571301305","0.552025799121","0.999998287810","0.999982614376","0.776012043466","0.810885586285","0.689617919831","0.599902676509","0.526272398156","0.712092028219","0.712084848565","0.000017385624","","","","",""
|
||||
"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","37842","506464","41477","351953","10594922057","41477","10594963534","0.000391","351953","858417","41.000236","0.924303894032","0.589997635182","0.999996085215","0.999962869327","0.794996860199","0.830219583690","0.720249040429","0.636004188257","0.562804063590","0.738469438547","0.738453718829","0.000037130673","","","","",""
|
||||
"SNOWBALL_GERMAN_LUCENE_FILTER","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","86669","723725","142783","621127","38436591110","142783","38436733893","0.000371","621127","1344852","46.185528","0.835220217240","0.538144717783","0.999996285246","0.999980126218","0.769070501515","0.752174652309","0.654551949931","0.579358576068","0.486493662760","0.670424751999","0.670415952491","0.000019873782","","","","",""
|
||||
"SNOWBALL_GERMAN_LUCENE_FILTER","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","46077","471644","34482","386773","10594929052","34482","10594963534","0.000325","386773","858417","45.056540","0.931870719939","0.549434598802","0.999996745435","0.999960243292","0.774715672118","0.817996747049","0.691284921032","0.598564290417","0.528216517210","0.715543160924","0.715527026594","0.000039756708","","","","",""
|
||||
"SNOWBALL_HUNGARIAN_DIRECT","HU_HU","hu-hu-default","1.0.0","359d46a01d751ec823705ad7f3dd1cc8f6663feb1a9d13cb04d0c6fb51ab646e","ALL_WORDS","PRIMARY_OUTPUT","19406","910688","1","910688","0","1","910688","116105","14275129","1281527","7842726","414652461946","1281527","414653743473","0.000309","7842726","22117855","35.458800","0.917621949087","0.645411998587","0.999996909404","0.999977996662","0.822704453996","0.846239680964","0.757813631609","0.686119053091","0.610064359819","0.769574048489","0.769564274829","0.000022003338","","","","",""
|
||||
@@ -241,6 +243,10 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_DIRECT","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4681","16906","23","16906","0","1","16906","6120","20847","1201","7458","142868459","1201","142869660","0.000841","7458","28305","26.348702","0.945527939042","0.736512983572","0.999991593737","0.999939404316","0.868252288654","0.894744070663","0.828034079399","0.770581364403","0.706534264217","0.834502009245","0.834474211808","0.000060595684","","","","",""
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","ALL_WORDS","PRIMARY_OUTPUT","4688","16937","23","16937","0","1","16937","6144","20854","1222","7508","143392932","1222","143394154","0.000852","7508","28362","26.472040","0.944645769161","0.735279599464","0.999991478035","0.999939130896","0.867635538749","0.893747964274","0.826916213966","0.769384020542","0.704908058410","0.833413920441","0.833385994629","0.000060869104","","","","",""
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4681","16906","23","16906","0","1","16906","6130","20824","1201","7481","142868459","1201","142869660","0.000841","7481","28305","26.429959","0.945471055619","0.735700406289","0.999991593737","0.999939243362","0.867846000013","0.894463296250","0.827498509835","0.769862102111","0.705754761743","0.834016450529","0.833988591624","0.000060756638","","","","",""
|
||||
"SNOWBALL_PERSIAN_DIRECT","FA_IR","fa-ir-default","1.0.0","b29a0d168a6a97f980666aa40b74a0edd8b6be4ab3320a7abfbb76b3529f4ea1","ALL_WORDS","PRIMARY_OUTPUT","69","3544","0","3544","0","1","3544","2029","6748","79","89296","6182073","79","6182152","0.001278","89296","96044","92.974054","0.988428299399","0.070259464412","0.999987221278","0.985764222716","0.535123342845","0.273526169012","0.131193436440","0.086290898024","0.070201720712","0.263526930171","0.261598150185","0.014235777284","","","","",""
|
||||
"SNOWBALL_PERSIAN_DIRECT","FA_IR","fa-ir-default","1.0.0","b29a0d168a6a97f980666aa40b74a0edd8b6be4ab3320a7abfbb76b3529f4ea1","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","69","3544","0","3544","0","1","3544","2029","6748","79","89296","6182073","79","6182152","0.001278","89296","96044","92.974054","0.988428299399","0.070259464412","0.999987221278","0.985764222716","0.535123342845","0.273526169012","0.131193436440","0.086290898024","0.070201720712","0.263526930171","0.261598150185","0.014235777284","","","","",""
|
||||
"SNOWBALL_POLISH_DIRECT","PL_PL","pl-pl-unimorph","1.0.0","8191ed727097839cc808cbc5c56a1bd78b3c851e7733ad226ad9a51519a54721","ALL_WORDS","PRIMARY_OUTPUT","9990","120867","1","120867","0","1","120867","19265","723037","70656","394036","7303167682","70656","7303238338","0.000967","394036","1117073","35.273970","0.910978174181","0.647260295433","0.999990325388","0.999936381518","0.823625310410","0.842338201869","0.756803292502","0.687038256475","0.608755869394","0.767880200391","0.767851723211","0.000063618482","","","","",""
|
||||
"SNOWBALL_POLISH_DIRECT","PL_PL","pl-pl-unimorph","1.0.0","8191ed727097839cc808cbc5c56a1bd78b3c851e7733ad226ad9a51519a54721","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","9846","119451","1","119451","0","1","119451","18927","718592","70647","392165","7133029571","70647","7133100218","0.000990","392165","1110757","35.306102","0.910487190826","0.646938979453","0.999990095891","0.999935127795","0.823464537672","0.841893538764","0.756414224030","0.686692785364","0.608252553741","0.767482673444","0.767453627968","0.000064872205","","","","",""
|
||||
"SNOWBALL_PORTUGUESE_DIRECT","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","ALL_WORDS","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","",""
|
||||
"SNOWBALL_PORTUGUESE_DIRECT","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","",""
|
||||
"SNOWBALL_PORTUGUESE_LUCENE_FILTER","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","ALL_WORDS","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","",""
|
||||
|
||||
|
@@ -1 +1 @@
|
||||
edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8 stemming-quality.csv
|
||||
d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5 stemming-quality.csv
|
||||
|
||||
@@ -52,8 +52,8 @@ Open [Language Benchmark Pages](languages/index.md) for the complete language li
|
||||
|
||||
The English dictionary coverage benchmark shows the current contracted-trie operating curve. With
|
||||
the full English dictionary, Radixor reaches `97.478%` all-token exactness and `97.197%`
|
||||
changed-token exactness at `98.0 ns/token`. Even with a deterministic 10% dictionary slice, it
|
||||
keeps `92.868%` all-token exactness and `76.516%` changed-token exactness at `80.6 ns/token`.
|
||||
changed-token exactness at `71.6 ns/token`. Even with a deterministic 10% dictionary slice, it
|
||||
keeps `92.868%` all-token exactness and `76.516%` changed-token exactness at `47.0 ns/token`.
|
||||
|
||||
Those figures should not be reduced to a single speed badge. The professional interpretation is a
|
||||
quality/speed envelope: the amount and quality of dictionary knowledge affect stemming precision,
|
||||
@@ -61,7 +61,17 @@ while contracted tries reduce lookup cost in uniform regions of the compiled gra
|
||||
|
||||
## Quality versus performance
|
||||
|
||||
Each language page keeps exact-root accuracy, JMH latency, and pairwise linguistic-quality results in separate tables. No undocumented scalar combines them. The 2026-07-23 language tables are generated from the unrounded JMH comparison report produced on the environment documented for this refresh. Readers should inspect the quality and speed dimensions side by side; no cross-language Pareto ranking is inferred from workloads with different dictionaries and token counts.
|
||||
Each language page keeps exact-root accuracy, JMH latency, and pairwise linguistic-quality results in separate tables. No undocumented scalar combines them. The 2026-08-10 language tables are generated exclusively from the current unrounded JMH comparison report produced on the environment documented for this refresh. The Snowball 3.1.0 matrix adds direct Czech, Persian, and Polish stemmers; all previously published Java stemmers were measured again in the same run. Readers should inspect the quality and speed dimensions side by side; no cross-language Pareto ranking is inferred from workloads with different dictionaries and token counts.
|
||||
|
||||
### New Snowball 3.1.0 rows
|
||||
|
||||
| New direct stemmer | All exact | Changed exact | Root preserved | Speed | Relative to same-language Radixor |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| Czech | 19.865% | 18.186% | 27.645% | 82.4 ns/token | 1.187× |
|
||||
| Persian | 3.660% | 0.000% | 100.000% | 298.1 ns/token | 6.486× |
|
||||
| Polish | 22.315% | 20.225% | 34.078% | 86.5 ns/token | 1.196× |
|
||||
|
||||
These rows describe exact agreement with each Radixor model dictionary and the measured direct API workload; they are not a universal linguistic ranking. In this dataset the three new Snowball stemmers are both less exact and slower than their same-language Radixor baseline. Lucene 10.5.0 does not expose the three new algorithms through `SnowballFilter`, so no synthetic Lucene wrapper rows were added.
|
||||
|
||||
<!-- STEMMING-QUALITY-OVERVIEW:START -->
|
||||
|
||||
@@ -76,16 +86,16 @@ The validated snapshot is a broad multilingual comparison covering the complete
|
||||
|
||||
| Language | Dictionary mode | Winner | Balanced accuracy | Runner-up | Difference | Exact tie | Deterministic stemmers |
|
||||
|---|---|---|---:|---|---:|---|---:|
|
||||
|Czech (`CS_CZ`)|ALL_WORDS|Radixor|0.996617|HUNSPELL CZECH LUCENE FILTER|0.142485045|no|3|
|
||||
|Czech (`CS_CZ`)|LOWERCASE_GROUPS_ONLY|Radixor|0.997195|HUNSPELL CZECH LUCENE FILTER|0.144045088|no|3|
|
||||
|Danish (`DA_DK`)|ALL_WORDS|Radixor|0.996243|SNOWBALL DANISH LUCENE FILTER|0.058337376|no|3|
|
||||
|Danish (`DA_DK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.996482|SNOWBALL DANISH DIRECT|0.058471663|no|3|
|
||||
|Czech (`CS_CZ`)|ALL_WORDS|Radixor|0.996617|HUNSPELL CZECH LUCENE FILTER|0.142485045|no|4|
|
||||
|Czech (`CS_CZ`)|LOWERCASE_GROUPS_ONLY|Radixor|0.997195|HUNSPELL CZECH LUCENE FILTER|0.144045088|no|4|
|
||||
|Danish (`DA_DK`)|ALL_WORDS|Radixor|0.996243|SNOWBALL DANISH DIRECT|0.053760569|no|3|
|
||||
|Danish (`DA_DK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.996482|SNOWBALL DANISH DIRECT|0.054099342|no|3|
|
||||
|Dutch (`NL_NL`)|ALL_WORDS|Radixor|0.988733|SNOWBALL DUTCH DIRECT|0.261639748|no|4|
|
||||
|Dutch (`NL_NL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.989114|SNOWBALL DUTCH DIRECT|0.258605347|no|4|
|
||||
|English (`US_UK`)|ALL_WORDS|Radixor|0.965537|ENGLISH LUCENE PORTER COPIED|0.010741250|no|11|
|
||||
|English (`US_UK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.966202|ENGLISH LUCENE PORTER COPIED|0.011138557|no|11|
|
||||
|Finnish (`FI_FI`)|ALL_WORDS|Radixor|0.984838|SNOWBALL FINNISH LUCENE FILTER|0.244558928|no|4|
|
||||
|Finnish (`FI_FI`)|LOWERCASE_GROUPS_ONLY|Radixor|0.988242|SNOWBALL FINNISH DIRECT|0.249897933|no|4|
|
||||
|Finnish (`FI_FI`)|LOWERCASE_GROUPS_ONLY|Radixor|0.988242|SNOWBALL FINNISH DIRECT|0.249699076|no|4|
|
||||
|French (`FR_FR`)|ALL_WORDS|Radixor|0.958627|SNOWBALL FRENCH DIRECT|0.109964908|no|6|
|
||||
|French (`FR_FR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.958856|SNOWBALL FRENCH DIRECT|0.110030565|no|6|
|
||||
|German (`DE_DE`)|ALL_WORDS|Radixor|0.910445|GERMAN CISTEM|0.031918024|no|8|
|
||||
@@ -100,10 +110,10 @@ The validated snapshot is a broad multilingual comparison covering the complete
|
||||
|Norwegian Bokmal (`NB_NO`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976240|SNOWBALL NORWEGIAN BOKMAL DIRECT|0.101954266|no|5|
|
||||
|Norwegian Nynorsk (`NN_NO`)|ALL_WORDS|Radixor|0.950991|SNOWBALL NORWEGIAN NYNORSK DIRECT|0.082896791|no|3|
|
||||
|Norwegian Nynorsk (`NN_NO`)|LOWERCASE_GROUPS_ONLY|Radixor|0.951104|SNOWBALL NORWEGIAN NYNORSK DIRECT|0.082851757|no|3|
|
||||
|Persian (`FA_IR`)|ALL_WORDS|Radixor|0.976360|PERSIAN LUCENE PERSIAN STEM FILTER|0.474147508|no|2|
|
||||
|Persian (`FA_IR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976360|PERSIAN LUCENE PERSIAN STEM FILTER|0.474147508|no|2|
|
||||
|Polish (`PL_PL`)|ALL_WORDS|Radixor|0.991105|POLISH LUCENE MORFOLOGIK FILTER|0.042712804|no|5|
|
||||
|Polish (`PL_PL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.991301|POLISH LUCENE MORFOLOGIK FILTER|0.042883749|no|5|
|
||||
|Persian (`FA_IR`)|ALL_WORDS|Radixor|0.976360|SNOWBALL PERSIAN DIRECT|0.441236451|no|3|
|
||||
|Persian (`FA_IR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976360|SNOWBALL PERSIAN DIRECT|0.441236451|no|3|
|
||||
|Polish (`PL_PL`)|ALL_WORDS|Radixor|0.991105|POLISH LUCENE MORFOLOGIK FILTER|0.042712804|no|6|
|
||||
|Polish (`PL_PL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.991301|POLISH LUCENE MORFOLOGIK FILTER|0.042883749|no|6|
|
||||
|Portuguese (`PT_PT`)|ALL_WORDS|Radixor|0.998542|SNOWBALL PORTUGUESE DIRECT|0.059619854|no|6|
|
||||
|Portuguese (`PT_PT`)|LOWERCASE_GROUPS_ONLY|Radixor|0.998542|SNOWBALL PORTUGUESE DIRECT|0.059619854|no|6|
|
||||
|Russian (`RU_RU`)|ALL_WORDS|Radixor|0.990188|SNOWBALL RUSSIAN LUCENE FILTER|0.155623602|no|4|
|
||||
@@ -166,7 +176,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|ITALIAN LUCENE ITALIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|NORWEGIAN BOKMAL LUCENE NORWEGIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|NORWEGIAN BOKMAL LUCENE NORWEGIAN MINIMAL STEM FILTER|1|0|0|0|5.000|5.000|
|
||||
|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|2.000|2.000|
|
||||
|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|3.000|3.000|
|
||||
|POLISH LUCENE MORFOLOGIK FILTER|1|0|0|1|2.000|2.000|
|
||||
|POLISH LUCENE STEMPEL DIRECT|1|0|0|0|4.000|4.000|
|
||||
|POLISH LUCENE STEMPEL FILTER|1|0|0|0|5.000|5.000|
|
||||
@@ -174,8 +184,9 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|PORTUGUESE LUCENE PORTUGUESE MINIMAL STEM FILTER|1|0|0|0|6.000|6.000|
|
||||
|PORTUGUESE LUCENE PORTUGUESE STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|RUSSIAN LUCENE RUSSIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|SNOWBALL DANISH DIRECT|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL DANISH LUCENE FILTER|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL CZECH DIRECT|1|0|0|0|4.000|4.000|
|
||||
|SNOWBALL DANISH DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL DANISH LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL DUTCH DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL DUTCH LUCENE FILTER|1|0|0|0|4.000|4.000|
|
||||
|SNOWBALL FINNISH DIRECT|1|0|0|1|3.000|3.000|
|
||||
@@ -192,6 +203,8 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|SNOWBALL NORWEGIAN BOKMAL LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL NORWEGIAN NYNORSK DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL NORWEGIAN NYNORSK LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL PERSIAN DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL POLISH DIRECT|1|0|0|0|6.000|6.000|
|
||||
|SNOWBALL PORTUGUESE DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL PORTUGUESE LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL RUSSIAN DIRECT|1|0|0|1|3.000|3.000|
|
||||
@@ -246,7 +259,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|ITALIAN LUCENE ITALIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|NORWEGIAN BOKMAL LUCENE NORWEGIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|NORWEGIAN BOKMAL LUCENE NORWEGIAN MINIMAL STEM FILTER|1|0|0|0|5.000|5.000|
|
||||
|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|2.000|2.000|
|
||||
|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|3.000|3.000|
|
||||
|POLISH LUCENE MORFOLOGIK FILTER|1|0|0|1|2.000|2.000|
|
||||
|POLISH LUCENE STEMPEL DIRECT|1|0|0|0|4.000|4.000|
|
||||
|POLISH LUCENE STEMPEL FILTER|1|0|0|0|5.000|5.000|
|
||||
@@ -254,6 +267,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|PORTUGUESE LUCENE PORTUGUESE MINIMAL STEM FILTER|1|0|0|0|6.000|6.000|
|
||||
|PORTUGUESE LUCENE PORTUGUESE STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|RUSSIAN LUCENE RUSSIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000|
|
||||
|SNOWBALL CZECH DIRECT|1|0|0|0|4.000|4.000|
|
||||
|SNOWBALL DANISH DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL DANISH LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL DUTCH DIRECT|1|0|0|1|2.000|2.000|
|
||||
@@ -272,6 +286,8 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa
|
||||
|SNOWBALL NORWEGIAN BOKMAL LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL NORWEGIAN NYNORSK DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL NORWEGIAN NYNORSK LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL PERSIAN DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL POLISH DIRECT|1|0|0|0|6.000|6.000|
|
||||
|SNOWBALL PORTUGUESE DIRECT|1|0|0|1|2.000|2.000|
|
||||
|SNOWBALL PORTUGUESE LUCENE FILTER|1|0|0|1|3.000|3.000|
|
||||
|SNOWBALL RUSSIAN DIRECT|1|0|0|1|2.000|2.000|
|
||||
@@ -305,7 +321,7 @@ These aggregates cover all 20 documented languages. Macro balanced accuracy give
|
||||
### Reproducible data
|
||||
|
||||
- [Machine-readable quality snapshot](data/stemming-quality.csv)
|
||||
- SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- [Linguistic quality methodology](reference/linguistic-quality.md)
|
||||
- [Tested stemmer inventory](reference/tested-stemmers.md)
|
||||
- [Reproducibility and raw data](reference/reproducibility.md)
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `cs-cz-default` | `1.0.0` | `CS_CZ` | 5,113 | 56,612 | 10,049 | 46,563 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `cs-cz-default` | `1.0.0` | `CS_CZ` | 5,113 | 56,612 | 10,049 | 46,563 | 46,563 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,14 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.465% | 99.439% | 99.582% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 99.465% | 99.439% | 99.582% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 84.850% | 82.269% | 96.806% | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene CzechStemFilter | 16.784% | 15.538% | 22.559% | Lucene Czech suffix stemmer implemented as a TokenFilter. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Official Snowball direct | 19.865% | 18.186% | 27.645% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -45,18 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `czechRadixor` | 3.395 | 0.066 | 72.9 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 381.189 | 32.563 | 8186.5 | 112.265 | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene CzechStemFilter | `czechLuceneCzechStemFilter` | 3.125 | 0.042 | 67.1 | 0.920 | Czech suffix stemmer implemented as a Lucene TokenFilter. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `czechRadixor` | 3.230 | 0.050 | 69.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 349.111 | 24.459 | 7497.6 | 108.091 | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene CzechStemFilter | `czechLuceneCzechStemFilter` | 2.927 | 0.032 | 62.9 | 0.906 | Czech suffix stemmer implemented as a Lucene TokenFilter. |
|
||||
| Official Snowball direct | `snowballDirect[CZECH]` | 3.835 | 0.320 | 82.4 | 1.187 | Official Snowball 3.1.0 generated Java stemmer; direct API. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -74,11 +66,11 @@ Runtime performance and linguistic grouping quality are independent dimensions.
|
||||
|
||||
The default model is `cs-cz-default`, loaded from classpath resource `org/egothor/stemmer/models/cs-cz-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.
|
||||
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996617** among 3 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.854132, a difference of 0.142485. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.997195** among 3 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.853150, a difference of 0.144045. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996617** among 4 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.854132, a difference of 0.142485. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.997195** among 4 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.853150, a difference of 0.144045. This rank does not imply leadership in throughput or every secondary metric.
|
||||
### `ALL_WORDS`
|
||||
|
||||
This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **8 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -89,6 +81,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|0.996617|0.000000%|0.676519%|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|0.854132|0.000691%|29.172837%|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|0.794343|0.000928%|41.130549%|
|
||||
|4|SNOWBALL CZECH DIRECT|0.786366|0.000904%|42.725842%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -99,6 +92,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.993235|1.000000|0.996617|0.999998|0.000002|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.958877|0.708272|0.999993|0.854132|0.999927|0.000073|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.935210|0.588695|0.999991|0.794343|0.999897|0.000103|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.935153|0.572742|0.999991|0.786366|0.999894|0.000106|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -109,6 +103,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.998640|0.996606|0.994581|0.993235|0.996612|0.996611|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.895506|0.814739|0.747335|0.687392|0.824103|0.824070|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.836710|0.722556|0.635811|0.565626|0.741992|0.741949|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.830101|0.710396|0.620864|0.550864|0.731848|0.731804|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -119,6 +114,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|298476|0|2033|1320705191|0 / 1320705191|2033 / 300509|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|212842|9128|87667|1320696063|9128 / 1320705191|87667 / 300509|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|176908|12256|123601|1320692935|12256 / 1320705191|123601 / 300509|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|172114|11935|128395|1320693256|11935 / 1320705191|128395 / 300509|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -193,7 +189,7 @@ Alternative candidates are capability analyses, not replacements for the determi
|
||||
|
||||
### `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **8 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -204,6 +200,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|0.997195|0.000000%|0.561033%|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|0.853150|0.000700%|29.369351%|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|0.792522|0.000918%|41.494586%|
|
||||
|4|SNOWBALL CZECH DIRECT|0.784821|0.000923%|43.034822%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -214,6 +211,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.994390|1.000000|0.997195|0.999999|0.000001|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.958957|0.706306|0.999993|0.853150|0.999925|0.000075|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.936557|0.585054|0.999991|0.792522|0.999895|0.000105|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.934577|0.569652|0.999991|0.784821|0.999891|0.000109|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -224,6 +222,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.998873|0.997187|0.995507|0.994390|0.997191|0.997190|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.894932|0.813466|0.745594|0.685581|0.822993|0.822960|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.836092|0.720206|0.632534|0.562751|0.740227|0.740184|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.828436|0.707849|0.617907|0.547807|0.729646|0.729601|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -234,6 +233,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
|1|Radixor|PRIMARY_OUTPUT|295818|0|1669|1284770069|0 / 1284770069|1669 / 297487|
|
||||
|2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|210117|8993|87370|1284761076|8993 / 1284770069|87370 / 297487|
|
||||
|3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|174046|11790|123441|1284758279|11790 / 1284770069|123441 / 297487|
|
||||
|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|169464|11863|128023|1284758206|11863 / 1284770069|128023 / 297487|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -330,7 +330,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `CS_CZ`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `da-dk-default` | `1.0.0` | `DA_DK` | 4,179 | 32,256 | 8,356 | 23,900 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `da-dk-default` | `1.0.0` | `DA_DK` | 4,179 | 32,256 | 8,356 | 23,900 | 23,900 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,14 +30,9 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.371% | 99.527% | 98.923% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 99.371% | 99.527% | 98.923% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SnowballFilter | 55.509% | 54.159% | 59.371% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 55.509% | 54.159% | 59.371% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Official Snowball direct | 55.971% | 54.791% | 59.347% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -45,18 +40,13 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `radixor[DANISH]` | 1.206 | 0.134 | 50.5 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[DANISH]` | 2.326 | 0.205 | 97.3 | 1.928 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[DANISH]` | 3.275 | 0.335 | 137.0 | 2.716 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `radixor[DANISH]` | 1.146 | 0.122 | 47.9 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[DANISH]` | 2.542 | 0.179 | 106.4 | 2.219 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[DANISH]` | 2.879 | 0.239 | 120.4 | 2.512 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -74,8 +64,8 @@ Runtime performance and linguistic grouping quality are independent dimensions.
|
||||
|
||||
The default model is `da-dk-default`, loaded from classpath resource `org/egothor/stemmer/models/da-dk-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.
|
||||
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996243** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH LUCENE FILTER` at 0.937905, a difference of 0.058337. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.996482** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.938010, a difference of 0.058472. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996243** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.942482, a difference of 0.053761. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.996482** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.942383, a difference of 0.054099. This rank does not imply leadership in throughput or every secondary metric.
|
||||
### `ALL_WORDS`
|
||||
|
||||
This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
@@ -87,8 +77,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.996243|0.000000%|0.751435%|
|
||||
|2|SNOWBALL DANISH LUCENE FILTER|0.937905|0.001273%|12.417638%|
|
||||
|3|SNOWBALL DANISH DIRECT|0.937839|0.001230%|12.431016%|
|
||||
|2|SNOWBALL DANISH DIRECT|0.942482|0.001236%|11.502313%|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|0.937905|0.001273%|12.417638%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -97,8 +87,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.992486|1.000000|0.996243|0.999998|0.000002|
|
||||
|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.940600|0.875824|0.999987|0.937905|0.999959|0.000041|
|
||||
|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942465|0.875690|0.999988|0.937839|0.999959|0.000041|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942799|0.884977|0.999988|0.942482|0.999961|0.000039|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.940600|0.875824|0.999987|0.937905|0.999959|0.000041|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -107,8 +97,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.998488|0.996229|0.993979|0.992486|0.996236|0.996235|
|
||||
|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.926889|0.907057|0.888055|0.829921|0.907634|0.907614|
|
||||
|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.928307|0.907851|0.888277|0.831252|0.908464|0.908444|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.930638|0.912973|0.895967|0.839881|0.913430|0.913411|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.926889|0.907057|0.888055|0.829921|0.907634|0.907614|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -117,8 +107,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|89021|0|674|389687465|0 / 389687465|674 / 89695|
|
||||
|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78557|4961|11138|389682504|4961 / 389687465|11138 / 89695|
|
||||
|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|78545|4795|11150|389682670|4795 / 389687465|11150 / 89695|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|79378|4816|10317|389682649|4816 / 389687465|10317 / 89695|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78557|4961|11138|389682504|4961 / 389687465|11138 / 89695|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -195,7 +185,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.996482|0.000000%|0.703596%|
|
||||
|2|SNOWBALL DANISH DIRECT|0.938010|0.001235%|12.396694%|
|
||||
|2|SNOWBALL DANISH DIRECT|0.942383|0.001240%|11.522225%|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|0.938010|0.001235%|12.396694%|
|
||||
|
||||
</div>
|
||||
@@ -205,7 +195,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.992964|1.000000|0.996482|0.999998|0.000002|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942392|0.876033|0.999988|0.938010|0.999959|0.000041|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942693|0.884778|0.999988|0.942383|0.999961|0.000039|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.942392|0.876033|0.999988|0.938010|0.999959|0.000041|
|
||||
|
||||
</details>
|
||||
@@ -215,7 +205,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.998585|0.996470|0.994363|0.992964|0.996476|0.996475|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.928328|0.908002|0.888547|0.831505|0.908607|0.908587|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.930511|0.912818|0.895784|0.839618|0.913277|0.913257|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.928328|0.908002|0.888547|0.831505|0.908607|0.908587|
|
||||
|
||||
</details>
|
||||
@@ -225,7 +215,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|88910|0|630|388404335|0 / 388404335|630 / 89540|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|78440|4795|11100|388399540|4795 / 388404335|11100 / 89540|
|
||||
|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|79223|4816|10317|388399519|4816 / 388404335|10317 / 89540|
|
||||
|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78440|4795|11100|388399540|4795 / 388404335|11100 / 89540|
|
||||
|
||||
</details>
|
||||
@@ -316,7 +306,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `DA_DK`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `nl-nl-default` | `1.0.0` | `NL_NL` | 4,992 | 31,466 | 9,981 | 21,485 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `nl-nl-default` | `1.0.0` | `NL_NL` | 4,992 | 31,466 | 9,981 | 21,485 | 21,485 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,35 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.120% | 98.711% | 100.000% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 99.120% | 98.711% | 100.000% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 46.590% | 22.718% | 97.976% | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Official Snowball direct | 15.954% | 8.992% | 30.939% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene SnowballFilter | 12.620% | 5.441% | 28.073% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `radixor[DUTCH]` | 1.410 | 0.139 | 65.6 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 24.183 | 2.889 | 1125.6 | 17.156 | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Official Snowball direct | `snowballDirect[DUTCH]` | 4.560 | 0.205 | 212.2 | 3.235 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[DUTCH]` | 7.762 | 0.262 | 361.3 | 5.506 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `radixor[DUTCH]` | 1.340 | 0.127 | 62.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 22.275 | 2.325 | 1036.8 | 16.621 | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Official Snowball direct | `snowballDirect[DUTCH]` | 4.298 | 0.185 | 200.0 | 3.207 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[DUTCH]` | 7.317 | 0.255 | 340.6 | 5.460 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -340,7 +330,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `NL_NL`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `us-uk-default` | `1.0.0` | `US_UK` | 396,939 | 1,004,374 | 793,874 | 210,500 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `us-uk-default` | `1.0.0` | `US_UK` | 396,939 | 1,004,374 | 793,874 | 210,500 | 210,500 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,47 +30,39 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 97.478% | 97.197% | 97.552% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 97.478% | 97.197% | 97.552% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene EnglishMinimalStemFilter | 90.981% | 65.189% | 97.820% | Minimal English plural reduction, not a full stemmer. |
|
||||
| Lucene KStemFilter | 80.076% | 76.608% | 80.996% | Krovetz-style English stemming TokenFilter; broader than minimal suffix reducers. |
|
||||
| Lucene HunspellStemFilter | 80.243% | 12.750% | 98.139% | Benchmark-only English Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene EnglishPossessiveFilter | 79.032% | 0.003% | 99.987% | Possessive-ending remover only, not a full stemmer. |
|
||||
| Snowball English / Porter2 | 40.342% | 46.296% | 38.763% | Porter2 rule-based suffix stemmer, distinct from original Porter. |
|
||||
| Snowball English / Porter2 | 40.346% | 46.302% | 38.767% | Porter2 rule-based suffix stemmer, distinct from original Porter. |
|
||||
| Lucene PorterStemFilter | 39.538% | 46.201% | 37.772% | Lucene TokenFilter path for Porter suffix rules; not dictionary-root equivalent. |
|
||||
| Lucene PorterStemmer direct copy | 39.538% | 46.201% | 37.772% | Direct Porter suffix-rule implementation generated under build for benchmark-only use. |
|
||||
| OpenNLP PorterStemmer | 39.538% | 46.201% | 37.772% | Apache OpenNLP Porter suffix-rule implementation. |
|
||||
| Snowball original Porter | 39.529% | 46.179% | 37.766% | Classic Porter rule-based suffix stemmer. |
|
||||
| Paice/Husk Lancaster | 28.055% | 37.039% | 25.673% | Aggressive Paice/Husk rule stemmer that often produces shorter stems. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `radixorUsUkProfiPreferredStem` | 17.489 | 1.380 | 83.1 | 1.000 | Full dictionary patch-command stemmer using compiled patch commands. |
|
||||
| Lucene EnglishPossessiveFilter | `luceneEnglishPossessiveFilter` | 17.151 | 0.215 | 81.5 | 0.981 | Possessive-ending remover only; not a full stemmer. |
|
||||
| Lucene EnglishMinimalStemFilter | `luceneEnglishMinimalStemFilter` | 18.522 | 0.152 | 88.0 | 1.059 | Narrow plural reduction filter; not a full stemmer. |
|
||||
| Lucene PorterStemmer direct copy | `lucenePorterStemmerCopied` | 17.651 | 0.129 | 83.9 | 1.009 | Benchmark-only generated copy of Lucene package-private Porter implementation. |
|
||||
| OpenNLP PorterStemmer | `opennlpPorterStemmer` | 17.681 | 0.139 | 84.0 | 1.011 | Apache OpenNLP Porter implementation. |
|
||||
| Snowball original Porter | `snowballOriginalPorter` | 33.290 | 1.916 | 158.1 | 1.904 | Classic Porter suffix-rule stemmer; historical English baseline, not a dictionary-equivalent stemmer. |
|
||||
| Lucene PorterStemFilter | `lucenePorterStemFilter` | 32.408 | 0.412 | 154.0 | 1.853 | Lucene TokenFilter integration path for Porter; includes TokenStream overhead. |
|
||||
| Lucene KStemFilter | `luceneKStemFilter` | 45.877 | 0.425 | 217.9 | 2.623 | Krovetz-style English TokenFilter; broader than minimal suffix filters. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 76.852 | 1.028 | 365.1 | 4.394 | Benchmark-only English Hunspell comparison using the benchmark Hunspell corpus. |
|
||||
| Snowball English / Porter2 | `snowballEnglishPorter2` | 46.568 | 2.414 | 221.2 | 2.663 | Porter2 suffix-rule stemmer, distinct from original Porter. |
|
||||
| Paice/Husk Lancaster | `paiceHuskLancaster` | 144.951 | 2.710 | 688.6 | 8.288 | Aggressive rule-based English stemmer. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `radixorUsUkProfiPreferredStem` | 14.397 | 0.915 | 68.4 | 1.000 | Full dictionary patch-command stemmer using compiled patch commands. |
|
||||
| Lucene EnglishPossessiveFilter | `luceneEnglishPossessiveFilter` | 15.034 | 0.322 | 71.4 | 1.044 | Possessive-ending remover only; not a full stemmer. |
|
||||
| Lucene EnglishMinimalStemFilter | `luceneEnglishMinimalStemFilter` | 16.352 | 0.244 | 77.7 | 1.136 | Narrow plural reduction filter; not a full stemmer. |
|
||||
| Lucene PorterStemmer direct copy | `lucenePorterStemmerCopied` | 16.491 | 0.149 | 78.3 | 1.145 | Benchmark-only generated copy of Lucene package-private Porter implementation. |
|
||||
| OpenNLP PorterStemmer | `opennlpPorterStemmer` | 16.481 | 0.175 | 78.3 | 1.145 | Apache OpenNLP Porter implementation. |
|
||||
| Snowball original Porter | `snowballOriginalPorter` | 30.634 | 1.620 | 145.5 | 2.128 | Classic Porter suffix-rule stemmer; historical English baseline, not a dictionary-equivalent stemmer. |
|
||||
| Lucene PorterStemFilter | `lucenePorterStemFilter` | 29.666 | 0.536 | 140.9 | 2.061 | Lucene TokenFilter integration path for Porter; includes TokenStream overhead. |
|
||||
| Lucene KStemFilter | `luceneKStemFilter` | 41.485 | 0.509 | 197.1 | 2.882 | Krovetz-style English TokenFilter; broader than minimal suffix filters. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 74.399 | 1.223 | 353.4 | 5.168 | Benchmark-only English Hunspell comparison using the benchmark Hunspell corpus. |
|
||||
| Snowball English / Porter2 | `snowballEnglishPorter2` | 43.117 | 1.983 | 204.8 | 2.995 | Porter2 suffix-rule stemmer, distinct from original Porter. |
|
||||
| Paice/Husk Lancaster | `paiceHuskLancaster` | 137.952 | 2.443 | 655.4 | 9.582 | Aggressive rule-based English stemmer. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -104,7 +96,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|0.954796|0.000207%|9.040545%|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|0.954796|0.000207%|9.040545%|
|
||||
|4|ENGLISH OPENNLP PORTER|0.954796|0.000207%|9.040545%|
|
||||
|5|ENGLISH SNOWBALL PORTER2|0.954708|0.000212%|9.058097%|
|
||||
|5|ENGLISH SNOWBALL PORTER2|0.954732|0.000212%|9.053310%|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|0.954659|0.000206%|9.067990%|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|0.952535|0.000960%|9.492110%|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|0.878645|0.000110%|24.270875%|
|
||||
@@ -122,7 +114,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.434174|0.909419|0.999998|0.954708|0.999998|0.000002|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.434309|0.909467|0.999998|0.954732|0.999998|0.000002|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.441440|0.909320|0.999998|0.954659|0.999998|0.000002|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.144284|0.905079|0.999990|0.952535|0.999990|0.000010|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.551014|0.757291|0.999999|0.878645|0.999998|0.000002|
|
||||
@@ -140,7 +132,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.484849|0.587747|0.746086|0.416176|0.628368|0.628367|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.484986|0.587880|0.746192|0.416310|0.628482|0.628481|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.492079|0.594348|0.750277|0.422827|0.633570|0.633569|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.173443|0.248891|0.440518|0.142133|0.361370|0.361368|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.582762|0.637891|0.704541|0.468312|0.645971|0.645970|
|
||||
@@ -158,7 +150,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|284971|371381|28384|175199052749|371381 / 175199424130|28384 / 313355|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|284986|371197|28369|175199052933|371197 / 175199424130|28369 / 313355|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|284940|360538|28415|175199063592|360538 / 175199424130|28415 / 313355|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|283611|1682034|29744|175197742096|1682034 / 175199424130|29744 / 313355|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|237301|193361|76054|175199230769|193361 / 175199424130|76054 / 313355|
|
||||
@@ -251,7 +243,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|0.955064|0.000222%|8.987032%|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|0.955064|0.000222%|8.987032%|
|
||||
|4|ENGLISH OPENNLP PORTER|0.955064|0.000222%|8.987032%|
|
||||
|5|ENGLISH SNOWBALL PORTER2|0.955016|0.000228%|8.996666%|
|
||||
|5|ENGLISH SNOWBALL PORTER2|0.955040|0.000228%|8.991849%|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|0.954926|0.000221%|9.014651%|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|0.952850|0.001032%|9.428933%|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|0.881028|0.000120%|23.794246%|
|
||||
@@ -269,7 +261,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.435017|0.910033|0.999998|0.955016|0.999998|0.000002|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.435153|0.910082|0.999998|0.955040|0.999998|0.000002|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.442235|0.909853|0.999998|0.954926|0.999998|0.000002|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.144700|0.905711|0.999990|0.952850|0.999990|0.000010|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.551013|0.762058|0.999999|0.881028|0.999998|0.000002|
|
||||
@@ -287,7 +279,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.485725|0.588647|0.746915|0.417080|0.629190|0.629189|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.485863|0.588782|0.747021|0.417215|0.629305|0.629304|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.492900|0.595181|0.751027|0.423671|0.634326|0.634325|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.173928|0.249533|0.441413|0.142553|0.362017|0.362015|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.583322|0.639575|0.707836|0.470129|0.648000|0.647999|
|
||||
@@ -305,7 +297,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output
|
||||
|2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382|
|
||||
|3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382|
|
||||
|4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|283368|368027|28014|161561621611|368027 / 161561989638|28014 / 311382|
|
||||
|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|283383|367843|27999|161561621795|367843 / 161561989638|27999 / 311382|
|
||||
|6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|283312|357325|28070|161561632313|357325 / 161561989638|28070 / 311382|
|
||||
|7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|282022|1666990|29360|161560322648|1666990 / 161561989638|29360 / 311382|
|
||||
|8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|237291|193354|74091|161561796284|193354 / 161561989638|74091 / 311382|
|
||||
@@ -408,7 +400,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `US_UK`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `fi-fi-default` | `1.0.0` | `FI_FI` | 57,027 | 1,865,215 | 110,525 | 1,754,690 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `fi-fi-default` | `1.0.0` | `FI_FI` | 57,027 | 1,865,215 | 110,525 | 1,754,690 | 1,754,690 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,14 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 98.661% | 98.803% | 96.408% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 98.661% | 98.803% | 96.408% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SnowballFilter | 10.991% | 10.268% | 22.471% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 10.991% | 10.268% | 22.471% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene FinnishLightStemFilter | 4.351% | 4.294% | 5.264% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Official Snowball direct | 10.995% | 10.272% | 22.462% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene FinnishLightStemFilter | 4.351% | 4.294% | 5.264% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -45,18 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `finnishRadixor` | 289.539 | 4.136 | 165.0 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene FinnishLightStemFilter | `finnishLuceneFinnishLightStemFilter` | 175.789 | 4.827 | 100.2 | 0.607 | Light Finnish suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[FINNISH]` | 259.889 | 8.924 | 148.1 | 0.898 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[FINNISH]` | 332.524 | 9.490 | 189.5 | 1.148 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `finnishRadixor` | 225.954 | 2.940 | 128.8 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene FinnishLightStemFilter | `finnishLuceneFinnishLightStemFilter` | 168.756 | 6.027 | 96.2 | 0.747 | Light Finnish suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[FINNISH]` | 247.984 | 16.083 | 141.3 | 1.097 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[FINNISH]` | 321.331 | 9.790 | 183.1 | 1.422 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -75,7 +67,7 @@ Runtime performance and linguistic grouping quality are independent dimensions.
|
||||
The default model is `fi-fi-default`, loaded from classpath resource `org/egothor/stemmer/models/fi-fi-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.
|
||||
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.984838** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH LUCENE FILTER` at 0.740279, a difference of 0.244559. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.988242** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH DIRECT` at 0.738344, a difference of 0.249898. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.988242** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH DIRECT` at 0.738543, a difference of 0.249699. This rank does not imply leadership in throughput or every secondary metric.
|
||||
### `ALL_WORDS`
|
||||
|
||||
This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
@@ -88,7 +80,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.984838|<0.000001%|3.032474%|
|
||||
|2|SNOWBALL FINNISH LUCENE FILTER|0.740279|0.000081%|51.944179%|
|
||||
|3|SNOWBALL FINNISH DIRECT|0.739671|0.000060%|52.065724%|
|
||||
|3|SNOWBALL FINNISH DIRECT|0.739870|0.000060%|52.025976%|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|0.695725|0.000094%|60.854936%|
|
||||
|
||||
</div>
|
||||
@@ -99,7 +91,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.999974|0.969675|1.000000|0.984838|0.999999|0.000001|
|
||||
|2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.921471|0.480558|0.999999|0.740279|0.999989|0.000011|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.940611|0.479343|0.999999|0.739671|0.999989|0.000011|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.940647|0.479740|0.999999|0.739870|0.999989|0.000011|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.890914|0.391451|0.999999|0.695725|0.999987|0.000013|
|
||||
|
||||
</details>
|
||||
@@ -110,7 +102,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.993763|0.984591|0.975587|0.969650|0.984708|0.984708|
|
||||
|2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.778598|0.631685|0.531413|0.461652|0.665448|0.665443|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.788800|0.635056|0.531468|0.465262|0.671472|0.671468|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.789035|0.635413|0.531862|0.465645|0.671764|0.671760|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.709787|0.543915|0.440884|0.373546|0.590550|0.590545|
|
||||
|
||||
</details>
|
||||
@@ -121,7 +113,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|30511413|804|954186|1599841738533|804 / 1599841739337|954186 / 31465599|
|
||||
|2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|15121052|1288634|16344547|1599840450703|1288634 / 1599841739337|16344547 / 31465599|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|15082807|952306|16382792|1599840787031|952306 / 1599841739337|16382792 / 31465599|
|
||||
|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|15095314|952479|16370285|1599840786858|952479 / 1599841739337|16370285 / 31465599|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|12317229|1508153|19148370|1599840231184|1508153 / 1599841739337|19148370 / 31465599|
|
||||
|
||||
</details>
|
||||
@@ -199,7 +191,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.988242|<0.000001%|2.351587%|
|
||||
|2|SNOWBALL FINNISH DIRECT|0.738344|0.000062%|52.331112%|
|
||||
|2|SNOWBALL FINNISH DIRECT|0.738543|0.000062%|52.291340%|
|
||||
|3|SNOWBALL FINNISH LUCENE FILTER|0.738344|0.000062%|52.331112%|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|0.694308|0.000077%|61.138333%|
|
||||
|
||||
@@ -210,7 +202,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.999973|0.976484|1.000000|0.988242|1.000000|0.000000|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.939951|0.476689|0.999999|0.738344|0.999989|0.000011|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.939988|0.477087|0.999999|0.738543|0.999989|0.000011|
|
||||
|3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.939951|0.476689|0.999999|0.738344|0.999989|0.000011|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.911893|0.388617|0.999999|0.694308|0.999987|0.000013|
|
||||
|
||||
@@ -221,7 +213,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.995185|0.988089|0.981093|0.976459|0.988159|0.988159|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.786987|0.632573|0.528815|0.462601|0.669376|0.669372|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.787224|0.632932|0.529209|0.462985|0.669668|0.669664|
|
||||
|3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.786987|0.632573|0.528815|0.462601|0.669376|0.669372|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.718421|0.544981|0.438999|0.374553|0.595296|0.595291|
|
||||
|
||||
@@ -232,7 +224,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|30037514|804|723369|1504706134249|804 / 1504706135053|723369 / 30760883|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|14663371|936765|16097512|1504705198288|936765 / 1504706135053|16097512 / 30760883|
|
||||
|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|14675605|936938|16085278|1504705198115|936938 / 1504706135053|16085278 / 30760883|
|
||||
|3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|14663371|936765|16097512|1504705198288|936765 / 1504706135053|16097512 / 30760883|
|
||||
|4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|11954192|1155011|18806691|1504704980042|1155011 / 1504706135053|18806691 / 30760883|
|
||||
|
||||
@@ -324,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `FI_FI`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `fr-fr-default` | `1.0.0` | `FR_FR` | 59,240 | 474,110 | 108,141 | 365,969 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `fr-fr-default` | `1.0.0` | `FR_FR` | 59,240 | 474,110 | 108,141 | 365,969 | 365,969 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,16 +30,12 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 94.831% | 94.859% | 94.734% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 94.831% | 94.859% | 94.734% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 68.923% | 63.617% | 86.876% | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene FrenchMinimalStemFilter | 11.472% | 6.236% | 29.192% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Lucene SnowballFilter | 8.551% | 5.183% | 19.952% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 8.462% | 5.067% | 19.952% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene FrenchLightStemFilter | 6.377% | 3.965% | 14.540% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Lucene FrenchLightStemFilter | 6.377% | 3.965% | 14.540% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -47,20 +43,16 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `frenchRadixor` | 49.340 | 0.986 | 134.8 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1781.070 | 43.544 | 4866.7 | 36.098 | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene FrenchMinimalStemFilter | `frenchLuceneFrenchMinimalStemFilter` | 19.093 | 0.681 | 52.2 | 0.387 | Minimal French suffix reducer; narrow baseline. |
|
||||
| Lucene FrenchLightStemFilter | `frenchLuceneFrenchLightStemFilter` | 29.553 | 0.465 | 80.8 | 0.599 | Light French suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[FRENCH]` | 121.376 | 0.865 | 331.7 | 2.460 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[FRENCH]` | 126.574 | 4.671 | 345.9 | 2.565 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `frenchRadixor` | 37.443 | 0.520 | 102.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1673.192 | 57.385 | 4572.0 | 44.686 | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene FrenchMinimalStemFilter | `frenchLuceneFrenchMinimalStemFilter` | 18.034 | 0.181 | 49.3 | 0.482 | Minimal French suffix reducer; narrow baseline. |
|
||||
| Lucene FrenchLightStemFilter | `frenchLuceneFrenchLightStemFilter` | 27.961 | 0.493 | 76.4 | 0.747 | Light French suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[FRENCH]` | 112.255 | 4.045 | 306.7 | 2.998 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[FRENCH]` | 119.555 | 4.560 | 326.7 | 3.193 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -358,7 +350,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `FR_FR`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `de-de-default` | `1.0.0` | `DE_DE` | 54,092 | 333,036 | 90,535 | 242,501 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `de-de-default` | `1.0.0` | `DE_DE` | 54,092 | 333,036 | 90,535 | 242,501 | 242,501 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,41 +30,33 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 92.725% | 92.847% | 92.396% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 92.725% | 92.847% | 92.396% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 47.064% | 29.661% | 93.678% | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| CISTEM (German) | 24.675% | 23.724% | 27.222% | Benchmark-only CISTEM implementation. |
|
||||
| Lucene GermanLightStemFilter | 37.434% | 35.465% | 42.707% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Lucene GermanLightStemFilter | 37.434% | 35.465% | 42.707% | Light suffix stemmer; intentionally narrower than Radixor's lexicon-trained transformation model. |
|
||||
| Lucene GermanMinimalStemFilter | 27.640% | 24.951% | 34.844% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Lucene SnowballFilter | 30.956% | 28.853% | 36.589% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 30.481% | 29.027% | 34.376% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Official Snowball direct | 30.483% | 29.030% | 34.376% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene GermanStemFilter | 21.559% | 19.312% | 27.576% | German Lucene stemming TokenFilter; broader than minimal/light variants. |
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `germanRadixor` | 40.571 | 1.647 | 167.3 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| CISTEM | `germanCistem` | 305.166 | 4.590 | 1258.4 | 7.522 | Benchmark-only CISTEM implementation. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 291.791 | 21.769 | 1203.3 | 7.192 | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene GermanMinimalStemFilter | `germanLuceneGermanMinimalStemFilter` | 23.903 | 0.208 | 98.6 | 0.589 | Minimal German suffix reduction; narrow baseline. |
|
||||
| Lucene GermanLightStemFilter | `germanLuceneGermanLightStemFilter` | 24.695 | 0.322 | 101.8 | 0.609 | Light German suffix stemmer; narrower than a dictionary stemmer. |
|
||||
| Lucene GermanStemFilter | `germanLuceneGermanStemFilter` | 72.140 | 1.544 | 297.5 | 1.778 | Older German stemming TokenFilter with normalization requirements. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[GERMAN]` | 110.086 | 2.315 | 454.0 | 2.713 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
| Official Snowball direct | `snowballDirect[GERMAN]` | 100.122 | 2.623 | 412.9 | 2.468 | Official Snowball generated Java stemmer; direct API. |
|
||||
|
||||
|
||||
|
||||
|
||||
| Radixor | `germanRadixor` | 27.697 | 0.583 | 114.2 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| CISTEM | `germanCistem` | 289.568 | 8.761 | 1194.1 | 10.455 | Benchmark-only CISTEM implementation. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 265.653 | 10.779 | 1095.5 | 9.591 | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene GermanMinimalStemFilter | `germanLuceneGermanMinimalStemFilter` | 22.385 | 0.217 | 92.3 | 0.808 | Minimal German suffix reduction; narrow baseline. |
|
||||
| Lucene GermanLightStemFilter | `germanLuceneGermanLightStemFilter` | 23.170 | 0.383 | 95.5 | 0.837 | Light German suffix stemmer; narrower than Radixor's lexicon-trained transformation model. |
|
||||
| Lucene GermanStemFilter | `germanLuceneGermanStemFilter` | 67.453 | 0.967 | 278.2 | 2.435 | Older German stemming TokenFilter with normalization requirements. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[GERMAN]` | 105.203 | 2.035 | 433.8 | 3.798 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
| Official Snowball direct | `snowballDirect[GERMAN]` | 91.847 | 2.301 | 378.7 | 3.316 | Official Snowball generated Java stemmer; direct API. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -96,7 +88,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.910445|0.000002%|17.910967%|
|
||||
|2|GERMAN CISTEM|0.878527|0.000674%|24.293900%|
|
||||
|3|SNOWBALL GERMAN DIRECT|0.776006|0.000171%|44.798684%|
|
||||
|3|SNOWBALL GERMAN DIRECT|0.776012|0.000171%|44.797420%|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|0.769071|0.000371%|46.185528%|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|0.753833|0.000191%|49.233299%|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|0.720992|0.000443%|55.801084%|
|
||||
@@ -111,7 +103,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.999400|0.820890|1.000000|0.910445|0.999994|0.000006|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|0.797231|0.757061|0.999993|0.878527|0.999985|0.000015|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.918570|0.552013|0.999998|0.776006|0.999983|0.000017|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.918571|0.552026|0.999998|0.776012|0.999983|0.000017|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.835220|0.538145|0.999996|0.769071|0.999980|0.000020|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.902792|0.507667|0.999998|0.753833|0.999981|0.000019|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.777304|0.441989|0.999996|0.720992|0.999976|0.000024|
|
||||
@@ -126,7 +118,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.957746|0.901392|0.851302|0.820486|0.905758|0.905755|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|0.788860|0.776627|0.764768|0.634824|0.776886|0.776879|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.810879|0.689608|0.599891|0.526260|0.712083|0.712076|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.810886|0.689618|0.599903|0.526272|0.712092|0.712085|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.752175|0.654552|0.579359|0.486494|0.670425|0.670416|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.781189|0.649884|0.556368|0.481355|0.676991|0.676984|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.674901|0.563540|0.483723|0.392311|0.586140|0.586130|
|
||||
@@ -141,7 +133,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|1103976|663|240876|38436733230|663 / 38436733893|240876 / 1344852|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|1018135|258954|326717|38436474939|258954 / 38436733893|326717 / 1344852|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|742376|65811|602476|38436668082|65811 / 38436733893|602476 / 1344852|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|742393|65811|602459|38436668082|65811 / 38436733893|602459 / 1344852|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|723725|142783|621127|38436591110|142783 / 38436733893|621127 / 1344852|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|682737|73514|662115|38436660379|73514 / 38436733893|662115 / 1344852|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|594410|170297|750442|38436563596|170297 / 38436733893|750442 / 1344852|
|
||||
@@ -231,7 +223,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.966959|0.000001%|6.608210%|
|
||||
|2|GERMAN CISTEM|0.914727|0.000812%|17.053716%|
|
||||
|3|SNOWBALL GERMAN DIRECT|0.794994|0.000391%|41.000819%|
|
||||
|3|SNOWBALL GERMAN DIRECT|0.794997|0.000391%|41.000236%|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|0.774716|0.000325%|45.056540%|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|0.768968|0.000130%|46.206331%|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|0.716147|0.000358%|56.770194%|
|
||||
@@ -246,7 +238,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.999900|0.933918|1.000000|0.966959|0.999995|0.000005|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|0.892172|0.829463|0.999992|0.914727|0.999978|0.000022|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.924303|0.589992|0.999996|0.794994|0.999963|0.000037|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.924304|0.589998|0.999996|0.794997|0.999963|0.000037|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.931871|0.549435|0.999997|0.774716|0.999960|0.000040|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.971001|0.537937|0.999999|0.768968|0.999961|0.000039|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.907196|0.432298|0.999996|0.716147|0.999950|0.000050|
|
||||
@@ -261,7 +253,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.985968|0.965783|0.946408|0.933831|0.966346|0.966343|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|0.878883|0.859676|0.841289|0.753887|0.860246|0.860236|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.830217|0.720244|0.635999|0.562799|0.738466|0.738450|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.830220|0.720249|0.636004|0.562804|0.738469|0.738454|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.817997|0.691285|0.598564|0.528217|0.715543|0.715527|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.836342|0.692324|0.590620|0.529431|0.722729|0.722714|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.743781|0.585563|0.482850|0.413990|0.626242|0.626223|
|
||||
@@ -276,7 +268,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|801691|80|56726|10594963454|80 / 10594963534|56726 / 858417|
|
||||
|2|GERMAN CISTEM|PRIMARY_OUTPUT|712025|86055|146392|10594877479|86055 / 10594963534|146392 / 858417|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|506459|41477|351958|10594922057|41477 / 10594963534|351958 / 858417|
|
||||
|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|506464|41477|351953|10594922057|41477 / 10594963534|351953 / 858417|
|
||||
|4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|471644|34482|386773|10594929052|34482 / 10594963534|386773 / 858417|
|
||||
|5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|461774|13791|396643|10594949743|13791 / 10594963534|396643 / 858417|
|
||||
|6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|371092|37962|487325|10594925572|37962 / 10594963534|487325 / 858417|
|
||||
@@ -378,7 +370,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `DE_DE`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ The default Hebrew model currently has no same-language third-party adapter in t
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `he-il-default` | `1.0.0` | `HE_IL` | 2,358 | 61,071 | 4,715 | 56,356 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `he-il-default` | `1.0.0` | `HE_IL` | 2,358 | 61,071 | 4,715 | 56,356 | 56,356 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -32,23 +32,17 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 98.228% | 98.172% | 98.897% | Full default-model Radixor dictionary patch-command stemmer. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `hebrewRadixor` | 3.921 | 0.140 | 69.6 | 1.000 | Full default-model Radixor dictionary patch-command stemmer. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `hebrewRadixor` | 3.570 | 0.074 | 63.3 | 1.000 | Full default-model Radixor dictionary patch-command stemmer. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the default language model used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Hebrew patch commands use forward traversal as declared by the model metadata.
|
||||
- Results are environment-specific and should be compared only with rows from the same benchmark run.
|
||||
|
||||
@@ -290,7 +284,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `HE_IL`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `hu-hu-default` | `1.0.0` | `HU_HU` | 19,406 | 935,713 | 38,775 | 896,938 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `hu-hu-default` | `1.0.0` | `HU_HU` | 19,406 | 935,713 | 38,775 | 896,938 | 896,938 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,13 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.222% | 99.537% | 91.948% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 99.222% | 99.537% | 91.948% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SnowballFilter | 66.445% | 66.938% | 55.043% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 66.445% | 66.938% | 55.043% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene HungarianLightStemFilter | 14.748% | 14.777% | 14.086% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
|
||||
|
||||
|
||||
| Lucene HungarianLightStemFilter | 14.748% | 14.777% | 14.086% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -44,17 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `hungarianRadixor` | 61.205 | 0.944 | 68.2 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HungarianLightStemFilter | `hungarianLuceneHungarianLightStemFilter` | 92.090 | 3.410 | 102.7 | 1.505 | Light Hungarian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[HUNGARIAN]` | 152.969 | 4.468 | 170.5 | 2.499 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[HUNGARIAN]` | 188.807 | 5.290 | 210.5 | 3.085 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `hungarianRadixor` | 52.020 | 1.337 | 58.0 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HungarianLightStemFilter | `hungarianLuceneHungarianLightStemFilter` | 87.362 | 3.444 | 97.4 | 1.679 | Light Hungarian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[HUNGARIAN]` | 158.081 | 9.757 | 176.2 | 3.039 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[HUNGARIAN]` | 179.398 | 8.041 | 200.0 | 3.449 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -322,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `HU_HU`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `it-it-default` | `1.0.0` | `IT_IT` | 10,009 | 337,546 | 20,004 | 317,542 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `it-it-default` | `1.0.0` | `IT_IT` | 10,009 | 337,546 | 20,004 | 317,542 | 317,542 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -29,31 +29,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.056% | 98.997% | 100.000% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene ItalianLightStemFilter | 0.466% | 0.479% | 0.270% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Radixor | 99.056% | 98.997% | 100.000% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene ItalianLightStemFilter | 0.466% | 0.479% | 0.270% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
| Lucene SnowballFilter | 0.041% | 0.043% | 0.010% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 0.041% | 0.043% | 0.010% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `italianRadixor` | 25.073 | 0.534 | 79.0 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene ItalianLightStemFilter | `italianLuceneItalianLightStemFilter` | 15.956 | 0.184 | 50.2 | 0.636 | Light Italian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[ITALIAN]` | 115.818 | 3.174 | 364.7 | 4.619 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[ITALIAN]` | 123.974 | 4.405 | 390.4 | 4.944 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `italianRadixor` | 22.503 | 0.434 | 70.9 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene ItalianLightStemFilter | `italianLuceneItalianLightStemFilter` | 15.008 | 0.278 | 47.3 | 0.667 | Light Italian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[ITALIAN]` | 109.401 | 2.983 | 344.5 | 4.862 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[ITALIAN]` | 116.392 | 3.271 | 366.5 | 5.172 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -321,7 +315,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `IT_IT`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `nb-no-default` | `1.0.0` | `NB_NO` | 17,929 | 90,757 | 33,376 | 57,381 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `nb-no-default` | `1.0.0` | `NB_NO` | 17,929 | 90,757 | 33,376 | 57,381 | 57,381 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,14 +30,11 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 96.852% | 97.637% | 95.503% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 96.852% | 97.637% | 95.503% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene NorwegianMinimalStemFilter | 57.107% | 53.913% | 62.599% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Official Snowball direct | 54.824% | 51.791% | 60.040% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene SnowballFilter | 54.803% | 51.780% | 60.001% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Lucene NorwegianLightStemFilter | 52.136% | 50.616% | 54.749% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
|
||||
|
||||
|
||||
| Lucene NorwegianLightStemFilter | 52.136% | 50.616% | 54.749% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -45,18 +42,15 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `norwegianBokmalRadixor` | 3.401 | 0.055 | 59.3 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene NorwegianMinimalStemFilter | `norwegianBokmalLuceneNorwegianMinimalStemFilter` | 2.943 | 0.023 | 51.3 | 0.865 | Minimal Norwegian suffix reducer. |
|
||||
| Lucene NorwegianLightStemFilter | `norwegianBokmalLuceneNorwegianLightStemFilter` | 3.358 | 0.036 | 58.5 | 0.987 | Light Norwegian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[NORWEGIAN_BOKMAL]` | 4.378 | 0.295 | 76.3 | 1.287 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_BOKMAL]` | 6.114 | 0.436 | 106.5 | 1.797 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `norwegianBokmalRadixor` | 3.240 | 0.087 | 56.5 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene NorwegianMinimalStemFilter | `norwegianBokmalLuceneNorwegianMinimalStemFilter` | 2.726 | 0.022 | 47.5 | 0.841 | Minimal Norwegian suffix reducer. |
|
||||
| Lucene NorwegianLightStemFilter | `norwegianBokmalLuceneNorwegianLightStemFilter` | 3.136 | 0.028 | 54.6 | 0.968 | Light Norwegian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[NORWEGIAN_BOKMAL]` | 4.711 | 0.410 | 82.1 | 1.454 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_BOKMAL]` | 5.681 | 0.206 | 99.0 | 1.753 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -332,7 +326,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `NB_NO`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `nn-no-default` | `1.0.0` | `NN_NO` | 4,688 | 19,651 | 6,089 | 13,562 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `nn-no-default` | `1.0.0` | `NN_NO` | 4,688 | 19,651 | 6,089 | 13,562 | 13,562 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,29 +30,23 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 93.089% | 91.395% | 96.863% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 93.089% | 91.395% | 96.863% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Official Snowball direct | 60.974% | 60.212% | 62.670% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene SnowballFilter | 60.918% | 60.146% | 62.638% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `radixor[NORWEGIAN_NYNORSK]` | 0.617 | 0.062 | 45.5 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[NORWEGIAN_NYNORSK]` | 0.955 | 0.076 | 70.4 | 1.548 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_NYNORSK]` | 1.352 | 0.106 | 99.7 | 2.191 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `radixor[NORWEGIAN_NYNORSK]` | 0.584 | 0.057 | 43.1 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[NORWEGIAN_NYNORSK]` | 1.087 | 0.097 | 80.2 | 1.861 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_NYNORSK]` | 1.256 | 0.095 | 92.6 | 2.149 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -312,7 +306,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `NN_NO`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `fa-ir-default` | `1.0.0` | `FA_IR` | 69 | 3,770 | 138 | 3,632 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `fa-ir-default` | `1.0.0` | `FA_IR` | 69 | 3,770 | 138 | 3,632 | 5,000 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -28,11 +28,9 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 95.836% | 95.677% | 100.000% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 95.836% | 95.677% | 100.000% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene PersianStemFilter | 1.485% | 0.000% | 40.580% | Lucene Persian suffix stemmer with required normalization in the measured path. |
|
||||
|
||||
|
||||
|
||||
| Official Snowball direct | 3.660% | 0.000% | 100.000% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -40,15 +38,13 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `persianRadixor` | 0.243 | 0.004 | 66.9 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene PersianStemFilter | `persianLucenePersianStemFilter` | 0.469 | 0.007 | 129.1 | 1.930 | Persian suffix stemmer with Lucene normalization in the measured path. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `persianRadixor` | 0.230 | 0.003 | 46.0 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene PersianStemFilter | `persianLucenePersianStemFilter` | 0.448 | 0.009 | 89.5 | 1.948 | Persian suffix stemmer with Lucene normalization in the measured path. |
|
||||
| Official Snowball direct | `snowballDirect[PERSIAN]` | 1.490 | 0.055 | 298.1 | 6.486 | Official Snowball 3.1.0 generated Java stemmer; direct API. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -66,11 +62,11 @@ Runtime performance and linguistic grouping quality are independent dimensions.
|
||||
|
||||
The default model is `fa-ir-default`, loaded from classpath resource `org/egothor/stemmer/models/fa-ir-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.
|
||||
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.976360** among 2 deterministic stemmers. The runner-up is `PERSIAN LUCENE PERSIAN STEM FILTER` at 0.502212, a difference of 0.474148. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.976360** among 2 deterministic stemmers. The runner-up is `PERSIAN LUCENE PERSIAN STEM FILTER` at 0.502212, a difference of 0.474148. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.976360** among 3 deterministic stemmers. The runner-up is `SNOWBALL PERSIAN DIRECT` at 0.535123, a difference of 0.441236. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.976360** among 3 deterministic stemmers. The runner-up is `SNOWBALL PERSIAN DIRECT` at 0.535123, a difference of 0.441236. This rank does not imply leadership in throughput or every secondary metric.
|
||||
### `ALL_WORDS`
|
||||
|
||||
This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -79,7 +75,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.976360|0.000000%|4.728041%|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%|
|
||||
|2|SNOWBALL PERSIAN DIRECT|0.535123|0.001278%|92.974054%|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -88,7 +85,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.952720|1.000000|0.976360|0.999277|0.000723|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.988428|0.070259|0.999987|0.535123|0.985764|0.014236|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -97,7 +95,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.990172|0.975787|0.961815|0.952720|0.976074|0.975715|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.273526|0.131193|0.086291|0.070202|0.263527|0.261598|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -106,7 +105,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|91503|0|4541|6182152|0 / 6182152|4541 / 96044|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|6748|79|89296|6182073|79 / 6182152|89296 / 96044|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -174,7 +174,7 @@ Alternative candidates are capability analyses, not replacements for the determi
|
||||
|
||||
### `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -183,7 +183,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |
|
||||
|---:|---|---:|---:|---:|
|
||||
|1|Radixor|0.976360|0.000000%|4.728041%|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%|
|
||||
|2|SNOWBALL PERSIAN DIRECT|0.535123|0.001278%|92.974054%|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -192,7 +193,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|1.000000|0.952720|1.000000|0.976360|0.999277|0.000723|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.988428|0.070259|0.999987|0.535123|0.985764|0.014236|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -201,7 +203,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|0.990172|0.975787|0.961815|0.952720|0.976074|0.975715|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.273526|0.131193|0.086291|0.070202|0.263527|0.261598|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -210,7 +213,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p
|
||||
| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |
|
||||
|---:|---|---|---:|---:|---:|---:|---:|---:|
|
||||
|1|Radixor|PRIMARY_OUTPUT|91503|0|4541|6182152|0 / 6182152|4541 / 96044|
|
||||
|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044|
|
||||
|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|6748|79|89296|6182073|79 / 6182152|89296 / 96044|
|
||||
|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -300,7 +304,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `FA_IR`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `pl-pl-unimorph` | `1.0.0` | `PL_PL` | 9,990 | 132,308 | 19,957 | 112,351 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `pl-pl-unimorph` | `1.0.0` | `PL_PL` | 9,990 | 132,308 | 19,957 | 112,351 | 112,351 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,14 +30,12 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 98.837% | 98.744% | 99.359% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 98.837% | 98.744% | 99.359% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 89.545% | 88.272% | 96.713% | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene MorfologikFilter | 87.729% | 86.606% | 94.047% | Dictionary-based path; Morfologik can emit multiple terms. |
|
||||
| Lucene StempelFilter | 70.009% | 69.262% | 74.220% | Lucene TokenFilter integration path for table-driven Polish Stempel. |
|
||||
| Lucene StempelStemmer direct | 70.009% | 69.262% | 74.220% | Direct table-driven Polish Stempel stemmer API. |
|
||||
|
||||
|
||||
|
||||
| Official Snowball direct | 22.315% | 20.225% | 34.078% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
## Speed
|
||||
|
||||
@@ -45,18 +43,16 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `polishRadixor` | 8.972 | 0.203 | 79.9 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 524.081 | 35.121 | 4664.7 | 58.412 | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene StempelStemmer direct | `polishLuceneStempelStemmerDirect` | 37.947 | 0.335 | 337.8 | 4.229 | Direct table-driven Polish Stempel stemmer API. |
|
||||
| Lucene StempelFilter | `polishLuceneStempelFilter` | 43.090 | 0.411 | 383.5 | 4.803 | Lucene TokenFilter integration path for table-driven Polish Stempel. |
|
||||
| Lucene MorfologikFilter | `polishLuceneMorfologikFilter` | 143.527 | 1.176 | 1277.5 | 15.997 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `polishRadixor` | 8.122 | 0.146 | 72.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 471.669 | 26.993 | 4198.2 | 58.070 | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene StempelStemmer direct | `polishLuceneStempelStemmerDirect` | 31.524 | 0.189 | 280.6 | 3.881 | Direct table-driven Polish Stempel stemmer API. |
|
||||
| Lucene StempelFilter | `polishLuceneStempelFilter` | 39.180 | 0.362 | 348.7 | 4.824 | Lucene TokenFilter integration path for table-driven Polish Stempel. |
|
||||
| Lucene MorfologikFilter | `polishLuceneMorfologikFilter` | 138.971 | 1.429 | 1236.9 | 17.110 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. |
|
||||
| Official Snowball direct | `snowballDirect[POLISH]` | 9.715 | 0.858 | 86.5 | 1.196 | Official Snowball 3.1.0 generated Java stemmer; direct API. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -74,11 +70,11 @@ Runtime performance and linguistic grouping quality are independent dimensions.
|
||||
|
||||
The default model is `pl-pl-unimorph`, loaded from classpath resource `org/egothor/stemmer/models/pl-pl-unimorph/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.
|
||||
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.991105** among 5 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948392, a difference of 0.042713. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.991301** among 5 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948417, a difference of 0.042884. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.991105** among 6 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948392, a difference of 0.042713. This rank does not imply leadership in throughput or every secondary metric.
|
||||
- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.991301** among 6 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948417, a difference of 0.042884. This rank does not imply leadership in throughput or every secondary metric.
|
||||
### `ALL_WORDS`
|
||||
|
||||
This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **12 result rows**, **6 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -91,6 +87,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|0.933457|0.000383%|13.308172%|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|0.855699|0.000602%|28.859618%|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|0.855699|0.000602%|28.859618%|
|
||||
|6|SNOWBALL POLISH DIRECT|0.823625|0.000967%|35.273970%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -103,6 +100,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.971931|0.866918|0.999996|0.933457|0.999976|0.000024|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.947549|0.711404|0.999994|0.855699|0.999950|0.000050|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.947549|0.711404|0.999994|0.855699|0.999950|0.000050|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.910978|0.647260|0.999990|0.823625|0.999936|0.000064|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -115,6 +113,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.948942|0.916426|0.886065|0.845744|0.917924|0.917913|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.888559|0.812669|0.748723|0.684450|0.821030|0.821007|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.888559|0.812669|0.748723|0.684450|0.821030|0.821007|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.842338|0.756803|0.687038|0.608756|0.767880|0.767852|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -127,6 +126,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|968411|27967|148662|7303210371|27967 / 7303238338|148662 / 1117073|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|794690|43990|322383|7303194348|43990 / 7303238338|322383 / 1117073|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|794690|43990|322383|7303194348|43990 / 7303238338|322383 / 1117073|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|723037|70656|394036|7303167682|70656 / 7303238338|394036 / 1117073|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -208,7 +208,7 @@ Alternative candidates are capability analyses, not replacements for the determi
|
||||
|
||||
### `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
This mode contains **12 result rows**, **6 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.
|
||||
|
||||
#### `PRIMARY_OUTPUT` ranking
|
||||
|
||||
@@ -221,6 +221,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|0.933546|0.000382%|13.290396%|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|0.856335|0.000611%|28.732387%|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|0.856335|0.000611%|28.732387%|
|
||||
|6|SNOWBALL POLISH DIRECT|0.823465|0.000990%|35.306102%|
|
||||
|
||||
</div>
|
||||
|
||||
@@ -233,6 +234,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.972469|0.867096|0.999996|0.933546|0.999975|0.000025|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.947796|0.712676|0.999994|0.856335|0.999949|0.000051|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.947796|0.712676|0.999994|0.856335|0.999949|0.000051|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.910487|0.646939|0.999990|0.823465|0.999935|0.000065|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -245,6 +247,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.949394|0.916764|0.886303|0.846320|0.918272|0.918260|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.889130|0.813590|0.749881|0.685758|0.821871|0.821848|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.889130|0.813590|0.749881|0.685758|0.821871|0.821848|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.841894|0.756414|0.686693|0.608253|0.767483|0.767454|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -257,6 +260,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output
|
||||
|3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|963133|27267|147624|7133072951|27267 / 7133100218|147624 / 1110757|
|
||||
|4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|791610|43601|319147|7133056617|43601 / 7133100218|319147 / 1110757|
|
||||
|5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|791610|43601|319147|7133056617|43601 / 7133100218|319147 / 1110757|
|
||||
|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|718592|70647|392165|7133029571|70647 / 7133100218|392165 / 1110757|
|
||||
|
||||
</details>
|
||||
|
||||
@@ -360,7 +364,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `PL_PL`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `pt-pt-default` | `1.0.0` | `PT_PT` | 4,001 | 215,490 | 8,002 | 207,488 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `pt-pt-default` | `1.0.0` | `PT_PT` | 4,001 | 215,490 | 8,002 | 207,488 | 207,488 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,35 +30,29 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.815% | 99.808% | 100.000% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene PortugueseLightStemFilter | 8.966% | 5.558% | 97.326% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Radixor | 99.815% | 99.808% | 100.000% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene PortugueseLightStemFilter | 8.966% | 5.558% | 97.326% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
| Lucene PortugueseMinimalStemFilter | 5.539% | 1.896% | 100.000% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Lucene SnowballFilter | 0.625% | 0.558% | 2.374% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 0.625% | 0.558% | 2.374% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene PortugueseStemFilter | 0.312% | 0.308% | 0.425% | Portuguese RSLP-style Lucene TokenFilter stemmer. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `portugueseRadixor` | 12.301 | 0.252 | 59.3 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene PortugueseLightStemFilter | `portugueseLucenePortugueseLightStemFilter` | 11.409 | 0.151 | 55.0 | 0.927 | Light Portuguese suffix stemmer. |
|
||||
| Lucene PortugueseMinimalStemFilter | `portugueseLucenePortugueseMinimalStemFilter` | 15.619 | 0.084 | 75.3 | 1.270 | Minimal Portuguese suffix reducer. |
|
||||
| Official Snowball direct | `snowballDirect[PORTUGUESE]` | 57.577 | 1.591 | 277.5 | 4.681 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[PORTUGUESE]` | 63.403 | 2.720 | 305.6 | 5.154 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
| Lucene PortugueseStemFilter | `portugueseLucenePortugueseStemFilter` | 158.014 | 5.150 | 761.6 | 12.845 | Portuguese RSLP-style Lucene TokenFilter. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `portugueseRadixor` | 10.902 | 0.166 | 52.5 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene PortugueseLightStemFilter | `portugueseLucenePortugueseLightStemFilter` | 10.125 | 0.161 | 48.8 | 0.929 | Light Portuguese suffix stemmer. |
|
||||
| Lucene PortugueseMinimalStemFilter | `portugueseLucenePortugueseMinimalStemFilter` | 14.338 | 0.156 | 69.1 | 1.315 | Minimal Portuguese suffix reducer. |
|
||||
| Official Snowball direct | `snowballDirect[PORTUGUESE]` | 52.193 | 1.905 | 251.5 | 4.788 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[PORTUGUESE]` | 58.991 | 2.457 | 284.3 | 5.411 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
| Lucene PortugueseStemFilter | `portugueseLucenePortugueseStemFilter` | 164.457 | 8.688 | 792.6 | 15.085 | Portuguese RSLP-style Lucene TokenFilter. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -342,7 +336,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `PT_PT`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `ru-ru-default` | `1.0.0` | `RU_RU` | 37,410 | 806,279 | 74,808 | 731,471 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `ru-ru-default` | `1.0.0` | `RU_RU` | 37,410 | 806,279 | 74,808 | 731,471 | 731,471 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,31 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 98.807% | 98.696% | 99.896% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene RussianLightStemFilter | 9.658% | 8.452% | 21.447% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Radixor | 98.807% | 98.696% | 99.896% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene RussianLightStemFilter | 9.658% | 8.452% | 21.447% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
| Lucene SnowballFilter | 9.162% | 8.162% | 18.936% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 9.162% | 8.162% | 18.936% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `russianRadixor` | 90.151 | 1.796 | 123.2 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene RussianLightStemFilter | `russianLuceneRussianLightStemFilter` | 59.456 | 2.102 | 81.3 | 0.660 | Light Russian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[RUSSIAN]` | 102.353 | 1.669 | 139.9 | 1.135 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[RUSSIAN]` | 138.597 | 4.727 | 189.5 | 1.537 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `russianRadixor` | 72.723 | 1.809 | 99.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene RussianLightStemFilter | `russianLuceneRussianLightStemFilter` | 58.844 | 3.217 | 80.4 | 0.809 | Light Russian suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[RUSSIAN]` | 103.471 | 8.136 | 141.5 | 1.423 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[RUSSIAN]` | 130.783 | 3.979 | 178.8 | 1.798 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -322,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `RU_RU`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `es-es-default` | `1.0.0` | `ES_ES` | 65,059 | 926,393 | 120,121 | 806,272 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `es-es-default` | `1.0.0` | `ES_ES` | 65,059 | 926,393 | 120,121 | 806,272 | 806,272 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,37 +30,31 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 97.459% | 97.544% | 96.891% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 97.459% | 97.544% | 96.891% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 49.074% | 42.656% | 92.154% | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene SpanishMinimalStemFilter | 17.284% | 5.347% | 97.403% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Lucene SpanishPluralStemFilter | 15.140% | 5.802% | 77.820% | Plural-focused suffix reducer; narrow baseline. |
|
||||
| Lucene SpanishLightStemFilter | 9.577% | 7.088% | 26.279% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Lucene SpanishLightStemFilter | 9.577% | 7.088% | 26.279% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
| Lucene SnowballFilter | 4.889% | 4.287% | 8.932% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 4.889% | 4.287% | 8.930% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `spanishRadixor` | 81.605 | 1.347 | 101.2 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 2033.430 | 14.863 | 2522.0 | 24.918 | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene SpanishMinimalStemFilter | `spanishLuceneSpanishMinimalStemFilter` | 42.144 | 1.556 | 52.3 | 0.516 | Minimal Spanish suffix reducer; narrow baseline. |
|
||||
| Lucene SpanishLightStemFilter | `spanishLuceneSpanishLightStemFilter` | 44.479 | 1.291 | 55.2 | 0.545 | Light Spanish suffix stemmer. |
|
||||
| Lucene SpanishPluralStemFilter | `spanishLuceneSpanishPluralStemFilter` | 96.537 | 3.418 | 119.7 | 1.183 | Plural-oriented Spanish suffix reducer. |
|
||||
| Official Snowball direct | `snowballDirect[SPANISH]` | 172.151 | 7.261 | 213.5 | 2.110 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[SPANISH]` | 201.697 | 9.363 | 250.2 | 2.472 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `spanishRadixor` | 63.164 | 1.885 | 78.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1936.800 | 18.685 | 2402.2 | 30.663 | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene SpanishMinimalStemFilter | `spanishLuceneSpanishMinimalStemFilter` | 40.414 | 1.475 | 50.1 | 0.640 | Minimal Spanish suffix reducer; narrow baseline. |
|
||||
| Lucene SpanishLightStemFilter | `spanishLuceneSpanishLightStemFilter` | 43.922 | 1.497 | 54.5 | 0.695 | Light Spanish suffix stemmer. |
|
||||
| Lucene SpanishPluralStemFilter | `spanishLuceneSpanishPluralStemFilter` | 89.799 | 3.474 | 111.4 | 1.422 | Plural-oriented Spanish suffix reducer. |
|
||||
| Official Snowball direct | `snowballDirect[SPANISH]` | 192.868 | 11.684 | 239.2 | 3.053 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[SPANISH]` | 182.719 | 6.957 | 226.6 | 2.893 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -366,7 +360,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `ES_ES`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `sv-se-default` | `1.0.0` | `SV_SE` | 12,371 | 110,468 | 24,731 | 85,737 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `sv-se-default` | `1.0.0` | `SV_SE` | 12,371 | 110,468 | 24,731 | 85,737 | 85,737 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,33 +30,27 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 96.713% | 97.407% | 94.307% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 96.713% | 97.407% | 94.307% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SwedishMinimalStemFilter | 49.532% | 49.186% | 50.730% | Minimal suffix reducer; narrow baseline, not a full stemmer. |
|
||||
| Lucene SwedishLightStemFilter | 45.672% | 46.383% | 43.209% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. |
|
||||
| Lucene SwedishLightStemFilter | 45.672% | 46.383% | 43.209% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. |
|
||||
| Official Snowball direct | 40.068% | 37.512% | 48.926% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
| Lucene SnowballFilter | 38.785% | 35.839% | 48.999% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `swedishRadixor` | 5.476 | 0.081 | 63.9 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene SwedishMinimalStemFilter | `swedishLuceneSwedishMinimalStemFilter` | 4.741 | 0.086 | 55.3 | 0.866 | Minimal Swedish suffix reducer. |
|
||||
| Lucene SwedishLightStemFilter | `swedishLuceneSwedishLightStemFilter` | 4.893 | 0.053 | 57.1 | 0.893 | Light Swedish suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[SWEDISH]` | 7.606 | 0.555 | 88.7 | 1.389 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[SWEDISH]` | 10.295 | 0.749 | 120.1 | 1.880 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `swedishRadixor` | 5.078 | 0.104 | 59.2 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SwedishMinimalStemFilter | `swedishLuceneSwedishMinimalStemFilter` | 4.417 | 0.061 | 51.5 | 0.870 | Minimal Swedish suffix reducer. |
|
||||
| Lucene SwedishLightStemFilter | `swedishLuceneSwedishLightStemFilter` | 5.090 | 0.373 | 59.4 | 1.002 | Light Swedish suffix stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[SWEDISH]` | 7.497 | 0.653 | 87.4 | 1.476 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[SWEDISH]` | 9.831 | 0.648 | 114.7 | 1.936 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -332,7 +326,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `SV_SE`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `uk-ua-default` | `1.0.0` | `UK_UA` | 1,493 | 15,737 | 2,985 | 12,752 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `uk-ua-default` | `1.0.0` | `UK_UA` | 1,493 | 15,737 | 2,985 | 12,752 | 12,752 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -30,31 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 99.307% | 99.365% | 99.062% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 99.307% | 99.365% | 99.062% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | 86.815% | 83.759% | 99.866% | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Lucene MorfologikFilter | 92.362% | 90.637% | 99.732% | Dictionary-based path; Morfologik can emit multiple terms. |
|
||||
| Morfologik direct | 92.362% | 90.637% | 99.732% | Direct dictionary lookup; first returned stem is used for quality when no ranking weight is exposed. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `ukrainianRadixor` | 0.639 | 0.009 | 50.1 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 47.919 | 5.308 | 3757.8 | 74.957 | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Morfologik direct | `ukrainianMorfologikDirect` | 8.662 | 0.105 | 679.3 | 13.550 | Direct Morfologik dictionary lookup; first returned stem is used for quality. |
|
||||
| Lucene MorfologikFilter | `ukrainianLuceneMorfologikFilter` | 15.367 | 0.219 | 1205.1 | 24.038 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `ukrainianRadixor` | 0.594 | 0.010 | 46.6 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 39.820 | 3.772 | 3122.6 | 67.067 | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. |
|
||||
| Morfologik direct | `ukrainianMorfologikDirect` | 8.231 | 0.121 | 645.5 | 13.863 | Direct Morfologik dictionary lookup; first returned stem is used for quality. |
|
||||
| Lucene MorfologikFilter | `ukrainianLuceneMorfologikFilter` | 14.700 | 0.176 | 1152.8 | 24.758 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -364,7 +358,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `UK_UA`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower
|
||||
|
||||
## Dictionary Corpus
|
||||
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: |
|
||||
| `yi-default` | `1.0.0` | `YI` | 802 | 4,300 | 1,524 | 2,776 |
|
||||
| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `yi-default` | `1.0.0` | `YI` | 802 | 4,300 | 1,524 | 2,776 | 5,000 |
|
||||
|
||||
## Radixor Patch Command Distribution
|
||||
|
||||
@@ -29,29 +29,23 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun
|
||||
|
||||
| Stemmer | All exact | Changed exact | Root preserved | Note |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| Radixor | 98.930% | 98.343% | 100.000% | Full Radixor dictionary patch-command stemmer. |
|
||||
| Radixor | 98.930% | 98.343% | 100.000% | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Lucene SnowballFilter | 2.837% | 2.558% | 3.346% | Lucene TokenFilter integration path around the Snowball algorithm. |
|
||||
| Official Snowball direct | 2.837% | 2.558% | 3.346% | Official Snowball generated Java stemmer; rule-based suffix algorithm. |
|
||||
|
||||
|
||||
|
||||
|
||||
## Speed
|
||||
|
||||
Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower.
|
||||
|
||||
| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| Radixor | `radixor[YIDDISH]` | 0.249 | 0.001 | 89.6 | 1.000 | Full Radixor dictionary patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[YIDDISH]` | 1.574 | 0.066 | 567.1 | 6.330 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[YIDDISH]` | 1.849 | 0.079 | 665.9 | 7.434 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
|
||||
|
||||
| Radixor | `radixor[YIDDISH]` | 0.234 | 0.001 | 46.8 | 1.000 | Radixor dictionary-trained patch-command stemmer. |
|
||||
| Official Snowball direct | `snowballDirect[YIDDISH]` | 1.487 | 0.062 | 297.5 | 6.354 | Official Snowball generated Java stemmer; direct API. |
|
||||
| Lucene SnowballFilter | `luceneSnowballFilter[YIDDISH]` | 1.754 | 0.070 | 350.8 | 7.492 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. |
|
||||
|
||||
## Interpretation Notes
|
||||
|
||||
- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie.
|
||||
- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource.
|
||||
- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work.
|
||||
- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs.
|
||||
- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available.
|
||||
@@ -311,7 +305,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated:
|
||||
### Provenance
|
||||
|
||||
- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`
|
||||
- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Evaluation command: `./gradlew stemmingQuality --no-daemon`
|
||||
- Dictionary language: `YI`
|
||||
- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`
|
||||
|
||||
@@ -6,16 +6,16 @@ This benchmark is the clearest demonstration of the Radixor quality/speed envelo
|
||||
|
||||
| Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token |
|
||||
| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 20.627 | 2.117 | 98.0 |
|
||||
| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.713 | 2.104 | 103.2 |
|
||||
| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 17.408 | 1.438 | 82.7 |
|
||||
| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 16.946 | 1.531 | 80.5 |
|
||||
| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 15.735 | 1.278 | 74.8 |
|
||||
| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 14.714 | 1.089 | 69.9 |
|
||||
| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.090 | 1.254 | 71.7 |
|
||||
| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 13.773 | 1.071 | 65.4 |
|
||||
| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 15.396 | 2.497 | 73.1 |
|
||||
| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 16.970 | 2.847 | 80.6 |
|
||||
| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 15.064 | 0.658 | 71.6 |
|
||||
| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 17.798 | 2.161 | 84.6 |
|
||||
| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 13.900 | 0.941 | 66.0 |
|
||||
| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 14.809 | 1.376 | 70.3 |
|
||||
| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 13.186 | 0.930 | 62.6 |
|
||||
| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 12.852 | 0.943 | 61.1 |
|
||||
| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 12.358 | 0.831 | 58.7 |
|
||||
| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 11.657 | 0.921 | 55.4 |
|
||||
| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 11.494 | 1.256 | 54.6 |
|
||||
| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 9.895 | 0.925 | 47.0 |
|
||||
|
||||
## Column Meanings
|
||||
|
||||
|
||||
@@ -4,25 +4,25 @@ The values below are environment-specific and must not be read as universal perf
|
||||
|
||||
| Item | Value |
|
||||
| --- | --- |
|
||||
| Benchmark date | 2026-07-23 (Europe/Prague) |
|
||||
| Benchmark date | 2026-08-10 (Europe/Prague) |
|
||||
| Corpus command | `./gradlew benchmarkCorpusReport --no-daemon` |
|
||||
| Exact-root accuracy command | Direct JMH execution of the four `*BenchmarkQuality` classes selected in `stemmer-accuracy-2026-07-23.txt`; timing scores are discarded |
|
||||
| Exact-root accuracy command | `tools/run-published-accuracy-benchmarks.sh 2026-08-10`; all four `*BenchmarkQuality` classes are selected and timing scores are discarded |
|
||||
| Stemming-quality command | `./gradlew stemmingQuality --no-daemon` |
|
||||
| Published speed command | `tools/run-published-speed-benchmarks.sh 2026-07-23` |
|
||||
| Published speed run interval | 2026-07-23 12:58:50 to 15:15:43 Europe/Prague (2 h 16 min 53 s, including idle intervals and both JMH suites) |
|
||||
| Published speed command | `tools/run-published-speed-benchmarks.sh 2026-08-10` |
|
||||
| Published speed run interval | 2026-08-10 16:22:33 to 18:36:41 Europe/Prague (2 h 14 min 8 s, including idle intervals and both JMH suites) |
|
||||
| Stabilization intervals | 120 s before the main speed matrix; 60 s between the main matrix and coverage-speed suite |
|
||||
| Corpus and command report | `build/reports/jmh/benchmark-corpora.csv` |
|
||||
| Exact-root reports | `build/reports/jmh/stemmer-accuracy-2026-07-23.csv` and `.txt` |
|
||||
| Speed reports | `build/reports/jmh/stemmer-speed-2026-07-23.csv` and `.txt` |
|
||||
| English coverage accuracy reports | `build/reports/jmh/english-coverage-accuracy-2026-07-23.csv` and `.txt` |
|
||||
| English coverage speed reports | `build/reports/jmh/english-coverage-speed-2026-07-23.csv` and `.txt` |
|
||||
| Exact-root reports | `build/reports/jmh/stemmer-accuracy-2026-08-10.csv` and `.txt` |
|
||||
| Speed reports | `build/reports/jmh/stemmer-speed-2026-08-10.csv` and `.txt` |
|
||||
| English coverage accuracy reports | `build/reports/jmh/english-coverage-accuracy-2026-08-10.csv` and `.txt` |
|
||||
| English coverage speed reports | `build/reports/jmh/english-coverage-speed-2026-08-10.csv` and `.txt` |
|
||||
| Stemming-quality reports | `build/reports/stemming-quality/stemming-quality.csv` and `.md` |
|
||||
| Environment report | `build/reports/jmh/performance-environment-2026-07-23.txt` |
|
||||
| Selected speed methods | `build/reports/jmh/published-speed-benchmarks-2026-07-23.txt` |
|
||||
| Environment report | `build/reports/jmh/performance-environment-2026-08-10.txt` |
|
||||
| Selected speed methods | `build/reports/jmh/published-speed-benchmarks-2026-08-10.txt` |
|
||||
| Comparison scope | Same-language methods used by the 20 language pages; `PolishPolimorfStemmerComparisonBenchmark`, all quality methods, the separate CISTEM gold-standard experiment, and internal trie microbenchmarks are excluded |
|
||||
| Model scope | Exactly the 20 IDs declared by `Language.defaultModelId()`; Polish uses `pl-pl-unimorph`, and `pl-pl-polimorf` is not measured |
|
||||
| Core base commit | `1f1b03c6a8d36a0918b92ebde698e5379a2a5946` |
|
||||
| Measured source state | `release@4.0.0-dirty`; exact tracked changes and untracked-source checksums are retained as `measured-source-2026-07-23.patch` and `measured-untracked-2026-07-23.sha256` |
|
||||
| Core base commit | `b45e143c8484c2ae1d6e31069a2e67232c4f7f29` |
|
||||
| Measured source state | `release@4.1.0-dirty`; exact tracked changes and untracked-source checksums are retained as `measured-source-2026-08-10.patch` and `measured-untracked-2026-08-10.sha256` |
|
||||
| JMH version | 1.37 |
|
||||
| Speed benchmark mode | Average time, `time/op` |
|
||||
| Score unit | `ns/op`; language pages additionally derive `ms/op` and `ns/token` |
|
||||
@@ -32,21 +32,21 @@ The values below are environment-specific and must not be read as universal perf
|
||||
| Speed threads | 1 |
|
||||
| Speed fork heap | Fixed `-Xms6g -Xmx6g` |
|
||||
| Reported uncertainty | JMH `Score Error (99.9%)` over 30 measured samples |
|
||||
| Observed relative uncertainty | Main speed matrix: maximum 11.945%, with 6 of 102 rows above 10%; coverage-speed curve: maximum 16.775%, with 3 of 10 rows above 10%; no published row exceeded 20% |
|
||||
| Observed relative uncertainty | Main speed matrix: maximum 10.607%, with 2 of 105 rows above 10%; coverage-speed curve: maximum 12.142%, with 2 of 10 rows above 10%; no published row exceeded 20% |
|
||||
| Deterministic measurements | Corpus, patch-command distribution, exact-root counters, coverage accuracy, and pairwise stemming quality are evaluated without interpreting runtime scores; no warmup is required |
|
||||
| JVM reported by JMH | JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3+9 |
|
||||
| Java runtime | OpenJDK Runtime Environment, Red Hat build 25.0.3+9 |
|
||||
| JVM reported by JMH | JDK 25.0.4, OpenJDK 64-Bit Server VM, 25.0.4+7 |
|
||||
| Java runtime | OpenJDK Runtime Environment, Red Hat build 25.0.4+7 |
|
||||
| JVM invoker | `/usr/lib/jvm/java-25-openjdk/bin/java` |
|
||||
| Operating system | Fedora Linux 44 (MATE-Compiz) |
|
||||
| Kernel | Linux 7.1.4-200.fc44.x86_64 |
|
||||
| Kernel | Linux 7.1.7-200.fc44.x86_64 |
|
||||
| Architecture | x86_64 |
|
||||
| CPU | AMD Ryzen 5 8600G w/ Radeon 760M Graphics |
|
||||
| CPU | AMD Ryzen 5 7600 6-Core Processor |
|
||||
| Physical / logical CPUs | 6 / 12 |
|
||||
| CPU frequency policy | `amd-pstate-epp`; governor `performance` on every logical CPU; EPP `performance`; boost enabled |
|
||||
| CPU affinity | Scheduler default; no explicit pinning |
|
||||
| Installed memory | 60 GiB reported by the operating system |
|
||||
| Pre-run idle state | Load average 0.25 / 0.36 / 0.71 after the 120 s idle interval; CPU Tctl 40.2 degrees Celsius; swap unused |
|
||||
| End-of-run state | Load average 1.16 / 1.28 / 1.32; CPU Tctl 60.5 degrees Celsius |
|
||||
| Installed memory | 61 GiB reported by the operating system |
|
||||
| Pre-run idle state | Load average 0.16 / 0.42 / 0.88 after the 120 s idle interval; CPU Tctl 57.4 degrees Celsius; swap unused |
|
||||
| End-of-run state | Load average 1.59 / 1.49 / 1.36; CPU Tctl 78.1 degrees Celsius |
|
||||
| Power and idle policy | Developer workstation on stable power; screensaver, suspend, and hibernation disabled |
|
||||
| Concurrent project work | None during the published speed and coverage-speed run |
|
||||
|
||||
@@ -67,15 +67,15 @@ The JMH runtime classpath contains the optional model artifact because it is a s
|
||||
Generated local report files for this benchmark update:
|
||||
|
||||
- `build/reports/jmh/benchmark-corpora.csv`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-07-23.csv`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-07-23.txt`
|
||||
- `build/reports/jmh/stemmer-speed-2026-07-23.csv`
|
||||
- `build/reports/jmh/stemmer-speed-2026-07-23.txt`
|
||||
- `build/reports/jmh/english-coverage-accuracy-2026-07-23.csv`
|
||||
- `build/reports/jmh/english-coverage-accuracy-2026-07-23.txt`
|
||||
- `build/reports/jmh/english-coverage-speed-2026-07-23.csv`
|
||||
- `build/reports/jmh/english-coverage-speed-2026-07-23.txt`
|
||||
- `build/reports/jmh/performance-environment-2026-07-23.txt`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-08-10.csv`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-08-10.txt`
|
||||
- `build/reports/jmh/stemmer-speed-2026-08-10.csv`
|
||||
- `build/reports/jmh/stemmer-speed-2026-08-10.txt`
|
||||
- `build/reports/jmh/english-coverage-accuracy-2026-08-10.csv`
|
||||
- `build/reports/jmh/english-coverage-accuracy-2026-08-10.txt`
|
||||
- `build/reports/jmh/english-coverage-speed-2026-08-10.csv`
|
||||
- `build/reports/jmh/english-coverage-speed-2026-08-10.txt`
|
||||
- `build/reports/jmh/performance-environment-2026-08-10.txt`
|
||||
- `build/reports/stemming-quality/stemming-quality.csv`
|
||||
- `build/reports/stemming-quality/stemming-quality.md`
|
||||
- `build/reports/stemming-quality/metric-correlations-pearson.csv`
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
The stemmer comparison suite measures Radixor and Java stemmers on the same language and deterministic Radixor model dictionary-derived data. Published Radixor rows in this refresh use contracted compiled patch tries, where uniform preferred-command subtrees are collapsed into accepting leaves before the trie is frozen for lookup. For each language, the registered default model resource stores the expected root as the first tab-separated field on a line and its surface forms on the same line. Every single-token field on that line can therefore be paired with the same expected root.
|
||||
|
||||
Published speed results come only from the exact method selection retained in `published-speed-benchmarks-2026-07-23.txt`. Internal `FrequencyTrie*` microbenchmarks, quality methods, the CISTEM gold-standard experiment, and the optional `PolishPolimorfStemmerComparisonBenchmark` are not part of those results.
|
||||
Published speed results come only from the exact method selection retained in `published-speed-benchmarks-2026-08-10.txt`. Internal `FrequencyTrie*` microbenchmarks, quality methods, the CISTEM gold-standard experiment, and the optional `PolishPolimorfStemmerComparisonBenchmark` are not part of those results. The Snowball 3.1.0 refresh adds direct Czech, Persian, and Polish workloads; the existing Radixor and Lucene workload domains are unchanged.
|
||||
|
||||
## Benchmark Passes
|
||||
|
||||
@@ -41,8 +41,8 @@ For right-to-left Radixor languages, patch application uses the traversal direct
|
||||
|
||||
The quality pass reports exact-root agreement against the expected root from the default-model dictionary line. External-stemmer counters are written to:
|
||||
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-07-23.csv`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-07-23.txt`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-08-10.csv`
|
||||
- `build/reports/jmh/stemmer-accuracy-2026-08-10.txt`
|
||||
|
||||
Accuracy is computed from standard JMH secondary rows:
|
||||
|
||||
|
||||
@@ -4,19 +4,28 @@
|
||||
|
||||
- Machine-readable CSV: [stemming-quality.csv](../data/stemming-quality.csv)
|
||||
- SHA-256 record: [stemming-quality.sha256](../data/stemming-quality.sha256)
|
||||
- SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8`
|
||||
- Complete scenarios: 308
|
||||
- SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5`
|
||||
- Complete scenarios: 314
|
||||
- Authoritative language universe: 20 languages
|
||||
- Language-page scenarios: 308 across 20 benchmark pages
|
||||
- Language-page scenarios: 314 across 20 benchmark pages
|
||||
|
||||
The CSV contains the model ID, independent model version, descriptor SHA-256, raw pair counts, raw over/under numerators and denominators, candidate statistics, and relation metrics. Reserved partition-metric columns remain empty because the gold standard is an overlapping cover. Documentation is regenerated from this file rather than manually transcribed. Publication fails when any row uses a model other than the language's registered default.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
./gradlew stemmingQuality
|
||||
./gradlew publishStemmingQualityDocumentation
|
||||
./gradlew verifyStemmingQualityDocumentation
|
||||
./gradlew --no-daemon stemmingQuality \
|
||||
publishStemmingQualityDocumentation \
|
||||
verifyStemmingQualityDocumentation
|
||||
./gradlew --no-daemon benchmarkCorpusReport writeJmhRuntimeClasspath
|
||||
tools/run-published-accuracy-benchmarks.sh 2026-08-10
|
||||
tools/run-published-speed-benchmarks.sh 2026-08-10
|
||||
python3 tools/update-benchmark-documentation.py \
|
||||
--corpus build/reports/jmh/benchmark-corpora.csv \
|
||||
--accuracy build/reports/jmh/stemmer-accuracy-2026-08-10.csv \
|
||||
--speed build/reports/jmh/stemmer-speed-2026-08-10.csv \
|
||||
--coverage-accuracy build/reports/jmh/english-coverage-accuracy-2026-08-10.csv \
|
||||
--coverage-speed build/reports/jmh/english-coverage-speed-2026-08-10.csv
|
||||
./gradlew test
|
||||
./gradlew prepareMkDocsSource
|
||||
mkdocs build --strict --config-file build/mkdocs/mkdocs.yml
|
||||
@@ -59,18 +68,19 @@ The Pages workflow publishes that staged documentation together with Javadoc, JU
|
||||
|
||||
## Performance benchmark reproduction
|
||||
|
||||
The current speed and coverage-speed command is:
|
||||
The current accuracy, speed, and coverage commands are:
|
||||
|
||||
```bash
|
||||
./gradlew writeJmhRuntimeClasspath --no-daemon
|
||||
tools/run-published-speed-benchmarks.sh 2026-07-23
|
||||
./gradlew --no-daemon benchmarkCorpusReport writeJmhRuntimeClasspath
|
||||
tools/run-published-accuracy-benchmarks.sh 2026-08-10
|
||||
tools/run-published-speed-benchmarks.sh 2026-08-10
|
||||
```
|
||||
|
||||
The runner refuses to start unless every CPU uses the `performance` governor, materializes the exact selected benchmark list, rejects quality/Polimorf/gold-standard methods, and requires the Hebrew speed path. It records hardware, JVM, source-state, JAR, classpath, corpus, quality, load, temperature, and governor provenance before running. The exact JMH configuration is listed in [Environment and reports](environment.md). Quality and performance reports are separate datasets and are not combined into an undocumented scalar.
|
||||
The speed runner refuses to start unless every CPU uses the `performance` governor, materializes the exact selected benchmark list, rejects quality/Polimorf/gold-standard methods, and requires the Hebrew speed path. It records hardware, JVM, source-state, JAR, classpath, corpus, quality, load, temperature, and governor provenance before running. The accuracy runner evaluates all four exact-root benchmark classes and verifies that every new Snowball 3.1.0 candidate exposes all six accuracy counters. The exact JMH configuration is listed in [Environment and reports](environment.md). Quality and performance reports are separate datasets and are not combined into an undocumented scalar.
|
||||
|
||||
## Recorded and unavailable provenance
|
||||
|
||||
The performance documentation records its 2026-07-23 environment, JDK 25.0.3, operating system, hardware, base revision, exact dirty patch, untracked-source checksums, executable JMH JAR checksum, and model descriptor checksums. The quality CSV embeds model identity and checksum in every row; run date, core source state, JVM, OS, and hardware are shared provenance on the environment page.
|
||||
The performance documentation records its 2026-08-10 environment, JDK, operating system, hardware, base revision, exact dirty patch, untracked-source checksums, executable JMH JAR checksum, and model descriptor checksums. The quality CSV embeds model identity and checksum in every row; run date, core source state, JVM, OS, and hardware are shared provenance on the environment page.
|
||||
|
||||
Exact immutable upstream revisions were not recorded for every legacy UniMorph import. That limitation remains explicit in model descriptors and cannot be repaired from filesystem timestamps. Dependency versions reproducible from repository configuration include Apache Lucene 10.5.0, Morfologik 2.1.9, the Ukrainian dictionary artifact 4.9.1, and JMH 1.37.
|
||||
|
||||
|
||||
@@ -4,10 +4,10 @@ The JMH adapter registry is authoritative for evaluated implementations and lang
|
||||
|
||||
| Family or implementation | Upstream / attribution | Tested version or revision | Evaluated scope | Output capability and adapter behaviour | Interpretation notes |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Radixor | Egothor / Radixor project | Base commit and measured working-tree state recorded on the environment page | All 20 reconciled default model languages; all 20 have benchmark pages | Deterministic preferred patch via `get`; ranked distinct alternatives via `getAll`; primary is always included | Model-dictionary-derived compiled patch trie. Default rows use each language's stable default model ID. |
|
||||
| Radixor | Egothor / Radixor project | Base commit and measured working-tree state recorded on the environment page | All 20 reconciled default model languages; all 20 have benchmark pages | Deterministic preferred patch via `get`; ranked distinct alternatives via `getAll`; primary is always included | Dictionary-trained compiled patch trie. Default rows use each language's stable default model ID. |
|
||||
| Apache Lucene language stem filters | Apache Lucene project | 10.5.0 | Adapter-declared language-specific subsets | TokenFilter lifecycle and language normalization match JMH; normally single-output | Light, minimal, possessive, and language stem filters deliberately implement different scopes. Narrow scope is not a defect. |
|
||||
| Apache Lucene SnowballFilter | Apache Lucene project using Snowball algorithms | Lucene 10.5.0 | Snowball-supported subset of Radixor languages | Single primary token emitted through the Lucene TokenFilter path | Includes TokenStream overhead and required normalization. |
|
||||
| Official Snowball Java | Snowball project | Repository preparation downloads the configured upstream Java distribution; an immutable revision was not recorded in the quality CSV | Same-language adapter subset | Direct generated Java API; single output | Rule-based suffix algorithms provide broad baselines rather than dictionary-root guarantees. |
|
||||
| Official Snowball Java | Snowball project | 3.1.0 source distribution; SHA-256 `5dab34d491f55f47b6e971569ffe6aadf5991512c648ddfe5d331b494cf6d655` | 17 same-language direct adapters, including the Czech, Persian, and Polish stemmers added in 3.1.0 | Direct generated Java API; single output | Rule-based suffix algorithms provide broad baselines rather than dictionary-root guarantees. Lucene 10.5.0 does not yet expose the three new algorithms through `SnowballFilter`, so those rows are direct-only. |
|
||||
| Lucene Stempel | Apache Lucene / Polish stemming tables | Lucene 10.5.0 | Polish | Direct and TokenFilter paths where registered; single primary output | Table-driven Polish implementation. |
|
||||
| Morfologik | Morfologik project; Lucene integration by Apache Lucene | Morfologik 2.1.9, Lucene integration 10.5.0; Ukrainian dictionary artifact 4.9.1 | Registered Polish and Ukrainian paths | Deterministic first lemma for primary comparison; all distinct lemma strings for candidate policies | Several analyses may share a lemma and are deduplicated by exact string equality. |
|
||||
| Hunspell via Lucene | Hunspell dictionaries from the `wooorm/dictionaries` repository; adapter by Apache Lucene | Lucene 10.5.0; dictionary repository revision was not recorded | Configured German, English, Spanish, French, Dutch, Polish, and Ukrainian dictionaries | First emitted stem is primary; all distinct stems at the token position are candidates | Dictionary content and affix rules differ by language. |
|
||||
|
||||
@@ -1,33 +1,42 @@
|
||||
# Built-in Languages and Default Models
|
||||
|
||||
“Supported language” means that Radixor defines a language enum value and publishes a corresponding default model artifact. It does not mean that a dictionary is embedded in the core JAR. Applications add model artifacts explicitly or use the optional standard pack.
|
||||
“Supported language” means that the repository maintains a default dictionary
|
||||
and runtime mapping for that language. Packaging differs by runtime: Java keeps
|
||||
the core dictionary-free and resolves external model artifacts, while the
|
||||
Python installs a separate `radixor-models-standard` data package containing
|
||||
the 20 default dictionaries in precompiled version 7 form.
|
||||
|
||||
The language enum carries language identity, writing direction, a legacy resource-directory name, and the stable default model ID. A model descriptor carries the independently versioned model identity and resource. See [Model Selection and Loading](model-selection-and-loading.md) for the API and the generated [model catalog](stemmer-model-catalog.md) for versions, provenance, checksums, and sizes.
|
||||
The Java language enum carries language identity, writing direction, a legacy
|
||||
resource-directory name, and the stable default model ID. A Java model
|
||||
descriptor carries the independently versioned model identity and resource.
|
||||
Python accepts the short alias or the same full model ID. See
|
||||
[Model Selection and Loading](model-selection-and-loading.md) for Java and
|
||||
[Python Usage and API](python/usage.md) for Python.
|
||||
|
||||
## Defaults and variants
|
||||
|
||||
| Language | Enum | Default model ID | Default artifact | Optional variants |
|
||||
|---|---|---|---|---|
|
||||
| Czech | `CS_CZ` | `cs-cz-default` | `org.egothor:radixor-model-cs-cz-default` | — |
|
||||
| Danish | `DA_DK` | `da-dk-default` | `org.egothor:radixor-model-da-dk-default` | — |
|
||||
| German | `DE_DE` | `de-de-default` | `org.egothor:radixor-model-de-de-default` | — |
|
||||
| Spanish | `ES_ES` | `es-es-default` | `org.egothor:radixor-model-es-es-default` | — |
|
||||
| Persian | `FA_IR` | `fa-ir-default` | `org.egothor:radixor-model-fa-ir-default` | — |
|
||||
| Finnish | `FI_FI` | `fi-fi-default` | `org.egothor:radixor-model-fi-fi-default` | — |
|
||||
| French | `FR_FR` | `fr-fr-default` | `org.egothor:radixor-model-fr-fr-default` | — |
|
||||
| Hebrew | `HE_IL` | `he-il-default` | `org.egothor:radixor-model-he-il-default` | — |
|
||||
| Hungarian | `HU_HU` | `hu-hu-default` | `org.egothor:radixor-model-hu-hu-default` | — |
|
||||
| Italian | `IT_IT` | `it-it-default` | `org.egothor:radixor-model-it-it-default` | — |
|
||||
| Norwegian Bokmål | `NB_NO` | `nb-no-default` | `org.egothor:radixor-model-nb-no-default` | — |
|
||||
| Dutch | `NL_NL` | `nl-nl-default` | `org.egothor:radixor-model-nl-nl-default` | — |
|
||||
| Norwegian Nynorsk | `NN_NO` | `nn-no-default` | `org.egothor:radixor-model-nn-no-default` | — |
|
||||
| Polish | `PL_PL` | `pl-pl-unimorph` | `org.egothor:radixor-model-pl-pl-unimorph` | `pl-pl-polimorf` / `org.egothor:radixor-model-pl-pl-polimorf` |
|
||||
| Portuguese | `PT_PT` | `pt-pt-default` | `org.egothor:radixor-model-pt-pt-default` | — |
|
||||
| Russian | `RU_RU` | `ru-ru-default` | `org.egothor:radixor-model-ru-ru-default` | — |
|
||||
| Swedish | `SV_SE` | `sv-se-default` | `org.egothor:radixor-model-sv-se-default` | — |
|
||||
| Ukrainian | `UK_UA` | `uk-ua-default` | `org.egothor:radixor-model-uk-ua-default` | — |
|
||||
| English | `US_UK` | `us-uk-default` | `org.egothor:radixor-model-us-uk-default` | — |
|
||||
| Yiddish | `YI` | `yi-default` | `org.egothor:radixor-model-yi-default` | — |
|
||||
| Language | Java enum | Python alias | Default model ID | Java default artifact | Optional variants |
|
||||
|---|---|---|---|---|---|
|
||||
| Czech | `CS_CZ` | `cs` | `cs-cz-default` | `org.egothor:radixor-model-cs-cz-default` | — |
|
||||
| Danish | `DA_DK` | `da` | `da-dk-default` | `org.egothor:radixor-model-da-dk-default` | — |
|
||||
| German | `DE_DE` | `de` | `de-de-default` | `org.egothor:radixor-model-de-de-default` | — |
|
||||
| Spanish | `ES_ES` | `es` | `es-es-default` | `org.egothor:radixor-model-es-es-default` | — |
|
||||
| Persian | `FA_IR` | `fa` | `fa-ir-default` | `org.egothor:radixor-model-fa-ir-default` | — |
|
||||
| Finnish | `FI_FI` | `fi` | `fi-fi-default` | `org.egothor:radixor-model-fi-fi-default` | — |
|
||||
| French | `FR_FR` | `fr` | `fr-fr-default` | `org.egothor:radixor-model-fr-fr-default` | — |
|
||||
| Hebrew | `HE_IL` | `he` | `he-il-default` | `org.egothor:radixor-model-he-il-default` | — |
|
||||
| Hungarian | `HU_HU` | `hu` | `hu-hu-default` | `org.egothor:radixor-model-hu-hu-default` | — |
|
||||
| Italian | `IT_IT` | `it` | `it-it-default` | `org.egothor:radixor-model-it-it-default` | — |
|
||||
| Norwegian Bokmål | `NB_NO` | `nb` | `nb-no-default` | `org.egothor:radixor-model-nb-no-default` | — |
|
||||
| Dutch | `NL_NL` | `nl` | `nl-nl-default` | `org.egothor:radixor-model-nl-nl-default` | — |
|
||||
| Norwegian Nynorsk | `NN_NO` | `nn` | `nn-no-default` | `org.egothor:radixor-model-nn-no-default` | — |
|
||||
| Polish | `PL_PL` | `pl` | `pl-pl-unimorph` | `org.egothor:radixor-model-pl-pl-unimorph` | `pl-pl-polimorf` / `org.egothor:radixor-model-pl-pl-polimorf` |
|
||||
| Portuguese | `PT_PT` | `pt` | `pt-pt-default` | `org.egothor:radixor-model-pt-pt-default` | — |
|
||||
| Russian | `RU_RU` | `ru` | `ru-ru-default` | `org.egothor:radixor-model-ru-ru-default` | — |
|
||||
| Swedish | `SV_SE` | `sv` | `sv-se-default` | `org.egothor:radixor-model-sv-se-default` | — |
|
||||
| Ukrainian | `UK_UA` | `uk` | `uk-ua-default` | `org.egothor:radixor-model-uk-ua-default` | — |
|
||||
| English | `US_UK` | `en` | `us-uk-default` | `org.egothor:radixor-model-us-uk-default` | — |
|
||||
| Yiddish | `YI` | `yi` | `yi-default` | `org.egothor:radixor-model-yi-default` | — |
|
||||
|
||||
The maintained table deliberately avoids duplicating mutable provenance and checksum fields. Those values come from module metadata and are generated into the model catalog.
|
||||
|
||||
@@ -44,6 +53,11 @@ The maintained table deliberately avoids duplicating mutable provenance and chec
|
||||
|
||||
UniMorph and PoliMorf have different lexical sources and provenance. Applications should compare outputs with application-specific regression tests before changing an explicit model choice.
|
||||
|
||||
In Python, `Stemmer("pl")` selects `pl-pl-unimorph`. The standard Python data
|
||||
package does not include PoliMorf; applications that need it must compile and
|
||||
load it explicitly as a trusted custom model. As in Java, it never changes the
|
||||
Polish default implicitly.
|
||||
|
||||
## Dependency patterns
|
||||
|
||||
Minimal English:
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
# CLI Compilation
|
||||
# Java CLI Compilation
|
||||
|
||||
Radixor provides a command-line compiler for turning line-oriented dictionary files into compact binary stemmer artifacts.
|
||||
|
||||
This page documents the Java CLI and its selectable reduction and normalization
|
||||
controls. Python exposes its production compilation profile through
|
||||
`radixor.compile(...)`; see [Compiling Dictionaries in Python](python/model-compilation.md).
|
||||
|
||||
The CLI output is not a model JAR. A model artifact contains a compressed textual dictionary, descriptor, index, checksum, and license so the runtime registry can discover and compile it. The CLI instead emits an already compiled binary trie for direct `loadBinaryCompiled(...)` use. Choose the model-module workflow when independently published classpath discovery is required; choose the CLI when the application owns a compiled binary asset.
|
||||
|
||||
This is the preferred preparation workflow when stemming should run against an already compiled artifact rather than against raw dictionary input. The CLI reads the dictionary, derives patch commands, builds a mutable trie, applies the selected subtree reduction strategy, and writes the final compiled trie in the project binary format under GZip compression. The result is a deployment-ready `.radixor.gz` file that can be loaded directly by application code.
|
||||
|
||||
@@ -8,11 +8,16 @@ Three artifacts must not be confused:
|
||||
|
||||
| Artifact | Representation | Consumer |
|
||||
|---|---|---|
|
||||
| Source textual dictionary | Plain UTF-8 tab-separated rows | Authors, parser, CLI, or model preparation |
|
||||
| Registered model resource | The same Radixor dictionary bytes under GZip, accompanied by index, descriptor, checksum, and license | `StemmerModelRegistry` and `StemmerPatchTrieLoader` |
|
||||
| Persisted compiled trie | GZip-compressed Radixor binary format, commonly `.radixor.gz` | `loadBinaryCompiled(...)` |
|
||||
| Source textual dictionary | Plain UTF-8 tab-separated rows, optionally GZip-compressed | Java and Python loaders or compilation tools |
|
||||
| Registered Java model resource | The same dictionary bytes under GZip, accompanied by index, descriptor, checksum, and license | Java `StemmerModelRegistry` and `StemmerPatchTrieLoader` |
|
||||
| Standard Python model resource | A precompiled version 7 `.rxc` artifact in the required `radixor-models-standard` data package | Python `Stemmer("<alias>")` |
|
||||
| Persisted compiled trie | GZip-compressed Radixor version 7 binary, commonly `.radixor.gz` in Java or `.rxc` in Python | Java `loadBinaryCompiled(...)` and Python `Stemmer(compiled=...)` |
|
||||
|
||||
The model file named `stemmer.gz` is not Java serialization and is not a pre-instantiated or persisted trie. It is compressed textual dictionary input parsed when the model is loaded.
|
||||
The Java model file named `stemmer.gz` is not Java serialization and is not a
|
||||
pre-instantiated or persisted trie. It is compressed textual dictionary input
|
||||
parsed when a Java model is loaded. Python's standard data distribution instead
|
||||
contains generated, precompiled version 7 tries; neither Python distribution
|
||||
ships the textual source dictionaries.
|
||||
|
||||
Consequently, compressed size is not a construction-memory estimate. The PoliMorf resource is 12,624,997 bytes compressed and 68,093,680 bytes decompressed, while full parsing, trie construction, reduction, and patch compilation require a dedicated verification JVM with a 6 GiB maximum heap.
|
||||
|
||||
@@ -25,6 +30,22 @@ generation are disclosed transformations; the in-memory trie is a Radixor runtim
|
||||
|
||||
Each logical line describes one canonical stem and zero or more known word variants that should reduce to that stem. The format is intentionally lightweight, easy to maintain in source control, and directly consumable both by the programmatic loader and by the CLI compiler.
|
||||
|
||||
## Use the format from either runtime
|
||||
|
||||
The dictionary semantics are shared; the integration entry points are not:
|
||||
|
||||
| Task | Java | Python |
|
||||
|---|---|---|
|
||||
| Load and compile text now | `StemmerPatchTrieLoader.loadCompiled(...)` | `Stemmer(path=...)` |
|
||||
| Compile a reusable binary | `org.egothor.stemmer.Compile` | `radixor.compile(...)` |
|
||||
| Load a compiled binary | `StemmerPatchTrieLoader.loadBinaryCompiled(...)` | `Stemmer(compiled=...)` |
|
||||
| Registered/standard language | External model JAR selected through the registry | Compiled model in `radixor-models-standard`, selected by alias or model ID |
|
||||
|
||||
Python compilation writes the shared version 7 stream using its fixed
|
||||
production reduction profile. Java exposes additional reduction and
|
||||
normalization controls. See [Java CLI Compilation](cli-compilation.md) and
|
||||
[Compiling Dictionaries in Python](python/model-compilation.md).
|
||||
|
||||
## Core structure
|
||||
|
||||
Each non-empty logical line has the following shape:
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# Fast Track
|
||||
# Java Fast Track
|
||||
|
||||
This page is the shortest path from an empty Java project to a working Radixor stemmer.
|
||||
It deliberately uses an external model artifact and the preferred compiled-command runtime API, so the
|
||||
first result does not require writing a dictionary, running the CLI compiler, or understanding
|
||||
reduction internals.
|
||||
|
||||
For the native Python package, start with the [Python Fast Track](python/fast-track.md)
|
||||
instead; Python does not use Maven model artifacts or the Java loader API.
|
||||
|
||||
Use this page when the goal is:
|
||||
|
||||
- add the dependency,
|
||||
|
||||
34
docs/getting-started.md
Normal file
34
docs/getting-started.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Get Started with Radixor
|
||||
|
||||
Radixor provides first-class Java and Python runtimes over the same learned
|
||||
patch-command model. Choose the environment that matches the application; both
|
||||
use the maintained language catalog and can exchange compiled version 7 tries.
|
||||
|
||||
## Python
|
||||
|
||||
Choose Python for native CPython integration, efficient batch calls, and a
|
||||
PyStemmer-compatible migration surface.
|
||||
|
||||
```bash
|
||||
python -m pip install radixor
|
||||
```
|
||||
|
||||
- [Python Fast Track](python/fast-track.md) — install and produce the first stem.
|
||||
- [Python Quick Start](python/quick-start.md) — build an application workflow.
|
||||
- [Python Usage and API](python/usage.md) — scalar, batch, cache, and custom-model APIs.
|
||||
|
||||
## Java
|
||||
|
||||
Choose Java for JVM integration, selectable reduction modes, registry-based
|
||||
model discovery, and extending or persisting compiled tries.
|
||||
|
||||
- [Java Fast Track](fast-track.md) — add the core/model dependencies and produce the first stem.
|
||||
- [Java Quick Start](quick-start.md) — loading, querying, extension, and persistence.
|
||||
- [Java Integration Deep Dive](integration-deep-dive.md) — lifecycle and production guidance.
|
||||
|
||||
## Shared concepts
|
||||
|
||||
- [Built-in Languages](built-in-languages.md) documents aliases, model IDs, and defaults.
|
||||
- [Dictionary Format](dictionary-format.md) defines the common source semantics.
|
||||
- [Benchmarks](benchmarks/index.md) separates runtime speed from linguistic quality.
|
||||
- [Architecture](architecture.md) explains shared behavior and runtime-specific internals.
|
||||
@@ -1,53 +1,9 @@
|
||||
<h1 class="visually-hidden">Home</h1>
|
||||
<p align="center">
|
||||
<img src="assets/images/banner.jpg" alt="Radixor banner" style="width: 100%; max-width: 1100px;">
|
||||
</p>
|
||||
---
|
||||
template: landing.html
|
||||
title: Radixor
|
||||
description: Learned transformation stemming for Java and Python using compact reduced tries of patch commands across 20 languages.
|
||||
---
|
||||
|
||||
**Radixor** is a high-performance, multi-language stemmer for Java, built for production-grade search and text-processing systems.
|
||||
# Radixor
|
||||
|
||||
It modernizes the proven Egothor patch-command trie approach and extends it for deployment realities that classic stemming pipelines do not handle well.
|
||||
|
||||
Traditional Egothor-style stemming workflows usually treat a compiled dictionary as a fixed artifact. Once built, its lexical knowledge is effectively closed unless the original source dictionary is recompiled. Radixor removes that constraint. An already compiled stemming structure can be extended with additional words and transformations, which makes it possible to evolve an existing dictionary for domain-specific, customer-specific, or deployment-specific vocabulary without rebuilding the entire lexical base from scratch.
|
||||
|
||||
Radixor also improves how ambiguous reductions can be handled at runtime. Instead of always forcing a single result, it can return multiple plausible stems when the input token cannot be reduced unambiguously. This allows downstream systems to preserve linguistic ambiguity where that is operationally useful, whether for retrieval quality, ranking strategies, diagnostics, or domain-specific normalization policies.
|
||||
|
||||
The project also has a clear research lineage. The historical idea behind this stemming family is described in Leo Galambos's paper *Lemmatizer for Document Information Retrieval Systems in JAVA* (SOFSEM 2001), which presents a semi-automatic stemming technique designed for Java-based information retrieval systems. In Radixor documentation, this reference serves as historical and algorithmic background rather than as technical documentation of the current implementation.
|
||||
|
||||
> Unlike traditional Egothor-based deployments, Radixor can extend an already compiled stemmer dictionary and can return multiple stems when a word is not reducible to a single unambiguous form.
|
||||
|
||||
Radixor delivers:
|
||||
|
||||
- **Fast runtime stemming** with compact lookup structures
|
||||
- **Multi-language adaptability** through dictionary-driven compilation
|
||||
- **Extension of compiled stemmer structures** without full recompilation from source dictionaries
|
||||
- **Incremental vocabulary growth** for deployment-specific lexical refinement
|
||||
- **Support for multiple stemming results** when reduction is ambiguous
|
||||
- **Deterministic behavior** suitable for reproducible processing pipelines
|
||||
- **Flexible integration paths**, including CLI-based and programmatic workflows
|
||||
- **Operational transparency** through continuously published quality and benchmark reports
|
||||
|
||||
Radixor is intended for teams that require consistent stemming quality at scale, while retaining the ability to evolve lexical resources after compilation and to handle ambiguous reductions with greater precision than traditional single-stem pipelines allow.
|
||||
|
||||
## Add the core and model data
|
||||
|
||||
The core `org.egothor:radixor` JAR contains no language dictionary. A minimal application adds one model; broad deployments may use the optional standard pack:
|
||||
|
||||
```groovy
|
||||
dependencies {
|
||||
implementation 'org.egothor:radixor:<radixor-version>'
|
||||
runtimeOnly 'org.egothor:radixor-model-pl-pl-unimorph:1.0.0'
|
||||
}
|
||||
```
|
||||
|
||||
`StemmerPatchTrieLoader.loadCompiled(Language.PL_PL, ...)` resolves the default `pl-pl-unimorph`. `pl-pl-polimorf` is a separate optional model selected by stable model ID. Follow [Model Selection and Loading](model-selection-and-loading.md) for runnable examples or choose artifacts from the generated [model catalog](stemmer-model-catalog.md).
|
||||
|
||||
## Start here
|
||||
|
||||
- Read [Fast Track](fast-track.md) when you want the shortest path to a working bundled stemmer.
|
||||
- Use [Model Selection and Loading](model-selection-and-loading.md) for default, explicit, dual-model, and ClassLoader examples.
|
||||
- Use [Integration Deep Dive](integration-deep-dive.md) when you are wiring Radixor into a real application or search pipeline.
|
||||
- Read [Quick Start](quick-start.md) for the broader developer walkthrough after the first result works.
|
||||
- Use [Built-in Languages](built-in-languages.md) to interpret language defaults and optional model variants.
|
||||
- Review [Benchmarking](benchmarking.md) and [Benchmark Results](benchmarks/index.md) for reproducible performance and quality methodology.
|
||||
- Open [CI Reports](reports.md) to inspect published build artifacts and quality metrics.
|
||||
- See the historical paper: [*Lemmatizer for Document Information Retrieval Systems in JAVA*](https://www.researchgate.net/publication/221512865_Lemmatizer_for_Document_Information_Retrieval_Systems_in_JAVA).
|
||||
Learned transformation stemming for Java and Python, using compact reduced tries of patch commands and reproducible quality/performance benchmarks across 20 languages.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Integration Deep Dive
|
||||
# Java Integration Deep Dive
|
||||
|
||||
This page explains how to integrate Radixor into a real Java application after the first
|
||||
fast-track experiment works. It covers dependencies, external model artifacts, runtime lifecycle,
|
||||
|
||||
133
docs/overrides/landing.html
Normal file
133
docs/overrides/landing.html
Normal file
@@ -0,0 +1,133 @@
|
||||
{% extends "base.html" %}
|
||||
{% block extrahead %}{{ super() }}<meta name="theme-color" content="#ffffff"><meta property="og:title" content="Radixor — High-quality multilingual stemming"><meta property="og:description" content="Learned transformation stemming for Java and Python: compact patch-command tries, 20 languages, and reproducible quality and performance benchmarks.">{% endblock %}
|
||||
{% block header %}<header class="rx2-header"><div class="rx2-header-inner"><a class="rx2-brand" href="{{ base_url }}/"><img src="{{ base_url }}/assets/images/radixor-logo.png" alt=""><span>RADIXOR</span></a><button class="rx2-menu" type="button" aria-label="Toggle menu" aria-expanded="false">☰</button><nav class="rx2-nav"><a href="{{ base_url }}/benchmarks/">Benchmarks</a><a href="{{ base_url }}/why-radixor-is-different/">Why Radixor</a><a href="{{ base_url }}/python/fast-track/">Python</a><a href="{{ base_url }}/fast-track/">Java</a><a href="{{ base_url }}/stemmer-models/">Models</a><a href="{{ base_url }}/getting-started/">Documentation</a><a href="https://github.com/leogalambos/Radixor">GitHub ↗</a></nav></div></header>{% endblock %}
|
||||
{% block tabs %}{% endblock %}
|
||||
{% block site_nav %}{% endblock %}
|
||||
{% block container %}
|
||||
<main class="rx2-page" id="main-content">
|
||||
<section class="rx2-hero">
|
||||
<div class="rx2-shell rx2-hero-top-grid">
|
||||
<div class="rx2-hero-copy">
|
||||
<div class="rx2-kicker"><span></span>Dictionary-trained. Not dictionary-bound.</div>
|
||||
<h1>High-quality multilingual stemming.<br><span>Built for <b>Java</b> and <em>Python</em>.</span></h1>
|
||||
<p>Radixor learns word-to-stem transformations from lexical evidence, compiles them into a reduced trie of patch commands, and applies those commands algorithmically at runtime. The training dictionary supplies evidence—it does not define a closed runtime vocabulary.</p>
|
||||
</div>
|
||||
|
||||
<div class="rx2-hero-actions" aria-label="Get Radixor and project assurances">
|
||||
<div class="rx2-action-eyebrow">Choose your runtime</div>
|
||||
<div class="rx2-cta-row">
|
||||
<a class="rx2-btn rx2-java" href="{{ base_url }}/quick-start/"><span class="rx2-btn-icon">♨</span>Get Radixor for Java <b>→</b></a>
|
||||
<a class="rx2-btn rx2-python" href="{{ base_url }}/python/fast-track/"><span class="rx2-python-glyph">Py</span>Get Radixor for Python <b>→</b></a>
|
||||
</div>
|
||||
<div class="rx2-assurances">
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="m7 9 9-5 9 5v14l-9 5-9-5Z" fill="none" stroke="currentColor" stroke-width="2.1"/><path d="m7 9 9 5 9-5M16 14v14" fill="none" stroke="currentColor" stroke-width="2.1"/></svg><span><b>Learned transformations</b><small>Trie + compact patch commands</small></span></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M5 24c4-9 9-13 22-16M8 9h7v7" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"/><path d="M4 27h24" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/></svg><span><b>Generalizes</b><small>Not restricted to listed word forms</small></span></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="m16 3 10 4v7c0 8-4 12-10 15C10 26 6 22 6 14V7Z" fill="none" stroke="currentColor" stroke-width="2.1"/><path d="m11 15 3 3 7-7" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round"/></svg><span><b>Reproducible</b><small>Public quality and speed evidence</small></span></div>
|
||||
</div>
|
||||
<div class="rx2-hero-links">
|
||||
<a href="{{ base_url }}/why-radixor-is-different/">Why Radixor is different →</a>
|
||||
<a href="{{ base_url }}/benchmarks/">See the benchmarks →</a>
|
||||
<a href="{{ base_url }}/getting-started/">Choose a documentation path →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="rx2-shell rx2-highlight-section">
|
||||
<aside class="rx2-highlight-card">
|
||||
<h2>Cross-platform highlights</h2>
|
||||
<div class="rx2-highlight-stats">
|
||||
<div class="rx2-highlight-stat purple"><i><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M10 4h12v6c0 7-3 10-6 10s-6-3-6-10V4Z" fill="none" stroke="currentColor" stroke-width="2.2"/><path d="M10 7H5c0 6 2 9 7 9M22 7h5c0 6-2 9-7 9M16 20v5M11 27h10" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/></svg></i><strong>20 / 20</strong><span>Java benchmark<br>wins</span></div>
|
||||
<div class="rx2-highlight-stat teal"><i><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M10 4h12v6c0 7-3 10-6 10s-6-3-6-10V4Z" fill="none" stroke="currentColor" stroke-width="2.2"/><path d="M10 7H5c0 6 2 9 7 9M22 7h5c0 6-2 9-7 9M16 20v5M11 27h10" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/></svg></i><strong>18 / 18</strong><span>Python wins vs<br>PyStemmer</span></div>
|
||||
<div class="rx2-highlight-stat blue"><i><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M5 23a12 12 0 1 1 22 0" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/><path d="m16 20 6-8" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/><circle cx="16" cy="20" r="2" fill="currentColor"/></svg></i><strong>1.67×</strong><span>Python geometric-<br>mean speedup</span></div>
|
||||
<div class="rx2-highlight-stat blue"><i><svg viewBox="0 0 32 32" aria-hidden="true"><circle cx="16" cy="16" r="12" fill="none" stroke="currentColor" stroke-width="2"/><path d="M4 16h24M16 4c4 4 5 8 5 12s-1 8-5 12c-4-4-5-8-5-12s1-8 5-12Z" fill="none" stroke="currentColor" stroke-width="2"/></svg></i><strong>20</strong><span>Supported<br>languages</span></div>
|
||||
</div>
|
||||
<div class="rx2-chart-title"><strong>Python native performance</strong> <span>(batch size N=100)</span></div>
|
||||
<div class="rx2-throughput"><b>3.66M – 5.99M</b> <span>words per second</span></div>
|
||||
<div class="rx2-chart-wrap">
|
||||
<div class="rx2-ylabels"><span>6M</span><span>4M</span><span>2M</span><span>0</span></div>
|
||||
<div class="rx2-chart" aria-label="Python throughput by language at batch size 100"><span class="rx2-bar" style="--h:57.1%" role="img" aria-label="Russian: 3.66 million words per second" title="Russian: 3.66M words/s"><img src="{{ base_url }}/assets/images/flags/ru.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:66.7%" role="img" aria-label="French: 4.27 million words per second" title="French: 4.27M words/s"><img src="{{ base_url }}/assets/images/flags/fr.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:67.7%" role="img" aria-label="German: 4.33 million words per second" title="German: 4.33M words/s"><img src="{{ base_url }}/assets/images/flags/de.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:68.3%" role="img" aria-label="Hebrew: 4.37 million words per second" title="Hebrew: 4.37M words/s"><img src="{{ base_url }}/assets/images/flags/he.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:68.6%" role="img" aria-label="Finnish: 4.39 million words per second" title="Finnish: 4.39M words/s"><img src="{{ base_url }}/assets/images/flags/fi.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:68.7%" role="img" aria-label="Yiddish: 4.39 million words per second" title="Yiddish: 4.39M words/s"><img src="{{ base_url }}/assets/images/flags/yi.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:69.7%" role="img" aria-label="Czech: 4.46 million words per second" title="Czech: 4.46M words/s"><img src="{{ base_url }}/assets/images/flags/cs.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:70.5%" role="img" aria-label="Ukrainian: 4.51 million words per second" title="Ukrainian: 4.51M words/s"><img src="{{ base_url }}/assets/images/flags/uk.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:74.4%" role="img" aria-label="Persian: 4.76 million words per second" title="Persian: 4.76M words/s"><img src="{{ base_url }}/assets/images/flags/fa.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:78.8%" role="img" aria-label="Hungarian: 5.04 million words per second" title="Hungarian: 5.04M words/s"><img src="{{ base_url }}/assets/images/flags/hu.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:80.3%" role="img" aria-label="Polish: 5.14 million words per second" title="Polish: 5.14M words/s"><img src="{{ base_url }}/assets/images/flags/pl.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:82.5%" role="img" aria-label="Swedish: 5.28 million words per second" title="Swedish: 5.28M words/s"><img src="{{ base_url }}/assets/images/flags/sv.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:83.5%" role="img" aria-label="Norwegian Bokmål: 5.34 million words per second" title="Norwegian Bokmål: 5.34M words/s"><img src="{{ base_url }}/assets/images/flags/nb.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:83.5%" role="img" aria-label="Dutch: 5.35 million words per second" title="Dutch: 5.35M words/s"><img src="{{ base_url }}/assets/images/flags/nl.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:84.8%" role="img" aria-label="Spanish: 5.43 million words per second" title="Spanish: 5.43M words/s"><img src="{{ base_url }}/assets/images/flags/es.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:86.6%" role="img" aria-label="English: 5.54 million words per second" title="English: 5.54M words/s"><img src="{{ base_url }}/assets/images/flags/en.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:87.6%" role="img" aria-label="Danish: 5.61 million words per second" title="Danish: 5.61M words/s"><img src="{{ base_url }}/assets/images/flags/da.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:91.5%" role="img" aria-label="Italian: 5.86 million words per second" title="Italian: 5.86M words/s"><img src="{{ base_url }}/assets/images/flags/it.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:92.6%" role="img" aria-label="Norwegian Nynorsk: 5.93 million words per second" title="Norwegian Nynorsk: 5.93M words/s"><img src="{{ base_url }}/assets/images/flags/nn.svg" alt=""></span>
|
||||
<span class="rx2-bar" style="--h:93.6%" role="img" aria-label="Portuguese: 5.99 million words per second" title="Portuguese: 5.99M words/s"><img src="{{ base_url }}/assets/images/flags/pt.svg" alt=""></span></div>
|
||||
</div>
|
||||
<div class="rx2-chart-range"><span><b>3.66M</b><small>Slowest</small></span><span><b>5.99M</b><small>Fastest</small></span></div>
|
||||
<p class="rx2-note"><b>*</b> The 2026-08-08 Python batch report contains 18 direct PyStemmer 3.1.0 comparisons, all won by Radixor. The 1.67× figure is the geometric mean across those measured rows; the largest measured advantage is 3.03× for Italian. PyStemmer has no direct Hebrew or Ukrainian comparator.</p>
|
||||
</aside>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="rx2-shell">
|
||||
<section class="rx2-java-banner">
|
||||
<img src="{{ base_url }}/assets/images/java-award.svg" alt="Java benchmark award">
|
||||
<div class="rx2-java-copy"><h2>Java: wins in all 20<br>benchmarked languages</h2><p>In the current same-language Java matrix, Radixor records the lowest measured runtime against every configured comparator on all 20 language pages.</p></div>
|
||||
<div class="rx2-win-box"><strong>20 / 20</strong><span>Benchmark wins</span><small>Across all 20 languages</small></div>
|
||||
<div class="rx2-verify"><b>Reproducible · Transparent · Verifiable</b><p>Every result is public, verifiable, and reproducible.</p><a href="{{ base_url }}/benchmarks/">View all benchmarks →</a></div>
|
||||
</section>
|
||||
|
||||
<h2 class="rx2-case-heading">Quality and speed you can trust — Finnish case study</h2>
|
||||
<section class="rx2-case-table">
|
||||
<div class="rx2-case-head"><span></span><span><img src="{{ base_url }}/assets/images/radixor-logo.png" alt=""> Radixor</span><span class="snow">❄ Snowball Finnish</span><span class="adv"><svg viewBox="0 0 32 32" aria-hidden="true"><circle cx="16" cy="16" r="12" fill="none" stroke="currentColor" stroke-width="2"/><path d="m10 16 4 4 8-9" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"/></svg> Radixor advantage</span></div>
|
||||
<div class="rx2-case-row"><span><i><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M5 23a12 12 0 1 1 22 0" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/><path d="m16 20 6-8" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/><circle cx="16" cy="20" r="2" fill="currentColor"/></svg></i><b>Quality score</b><small>(higher is better)</small></span><strong>0.9848</strong><span>~0.740</span><b class="green">+33% <small>higher quality</small></b></div>
|
||||
<div class="rx2-case-row"><span><i class="dots">◌</i><b>Overstemming</b><small>(lower is better)</small></span><strong>3.03%</strong><b class="red">~52%</b><b class="green">94.2% <small>lower</small></b></div>
|
||||
<div class="rx2-case-row"><span><i><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M6 26V16M13 26V11M20 26V6M27 26V3" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"/></svg></i><b>Speed</b><small>(vs. PyStemmer)</small></span><strong>1.14× <small>faster</small></strong><span>1.00× <small>(baseline)</small></span><b class="green">1.14× <small>faster</small></b></div>
|
||||
</section>
|
||||
<p class="rx2-case-note">Quality comparator: published Snowball Finnish Lucene/Snowball results. Speed comparator: PyStemmer 3.1.0 at batch size N=100. The point is the combined quality/performance envelope, not wrapper identity.</p>
|
||||
|
||||
<section class="rx2-languages-strip">
|
||||
<div class="rx2-language-label"><svg viewBox="0 0 32 32" aria-hidden="true"><circle cx="16" cy="16" r="12" fill="none" stroke="currentColor" stroke-width="2"/><path d="M4 16h24M16 4c4 4 5 8 5 12s-1 8-5 12c-4-4-5-8-5-12s1-8 5-12Z" fill="none" stroke="currentColor" stroke-width="2"/></svg><div><strong>20 languages</strong><span>Full Unicode support</span></div></div>
|
||||
<div class="rx2-language-list"><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/czech/" aria-label="Open Czech benchmark"><img src="{{ base_url }}/assets/images/flags/cs.svg" alt="Czech flag"><span>Czech</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/danish/" aria-label="Open Danish benchmark"><img src="{{ base_url }}/assets/images/flags/da.svg" alt="Danish flag"><span>Danish</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/dutch/" aria-label="Open Dutch benchmark"><img src="{{ base_url }}/assets/images/flags/nl.svg" alt="Dutch flag"><span>Dutch</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/english/" aria-label="Open English benchmark"><img src="{{ base_url }}/assets/images/flags/en.svg" alt="English flag"><span>English</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/finnish/" aria-label="Open Finnish benchmark"><img src="{{ base_url }}/assets/images/flags/fi.svg" alt="Finnish flag"><span>Finnish</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/french/" aria-label="Open French benchmark"><img src="{{ base_url }}/assets/images/flags/fr.svg" alt="French flag"><span>French</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/german/" aria-label="Open German benchmark"><img src="{{ base_url }}/assets/images/flags/de.svg" alt="German flag"><span>German</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/hebrew/" aria-label="Open Hebrew benchmark"><img src="{{ base_url }}/assets/images/flags/he.svg" alt="Hebrew flag"><span>Hebrew</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/hungarian/" aria-label="Open Hungarian benchmark"><img src="{{ base_url }}/assets/images/flags/hu.svg" alt="Hungarian flag"><span>Hungarian</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/italian/" aria-label="Open Italian benchmark"><img src="{{ base_url }}/assets/images/flags/it.svg" alt="Italian flag"><span>Italian</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/norwegian-bokmal/" aria-label="Open Norwegian Bokmål benchmark"><img src="{{ base_url }}/assets/images/flags/nb.svg" alt="Norwegian Bokmål flag"><span>Norwegian Bokmål</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/norwegian-nynorsk/" aria-label="Open Norwegian Nynorsk benchmark"><img src="{{ base_url }}/assets/images/flags/nn.svg" alt="Norwegian Nynorsk flag"><span>Norwegian Nynorsk</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/persian/" aria-label="Open Persian benchmark"><img src="{{ base_url }}/assets/images/flags/fa.svg" alt="Persian flag"><span>Persian</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/polish/" aria-label="Open Polish benchmark"><img src="{{ base_url }}/assets/images/flags/pl.svg" alt="Polish flag"><span>Polish</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/portuguese/" aria-label="Open Portuguese benchmark"><img src="{{ base_url }}/assets/images/flags/pt.svg" alt="Portuguese flag"><span>Portuguese</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/russian/" aria-label="Open Russian benchmark"><img src="{{ base_url }}/assets/images/flags/ru.svg" alt="Russian flag"><span>Russian</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/spanish/" aria-label="Open Spanish benchmark"><img src="{{ base_url }}/assets/images/flags/es.svg" alt="Spanish flag"><span>Spanish</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/swedish/" aria-label="Open Swedish benchmark"><img src="{{ base_url }}/assets/images/flags/sv.svg" alt="Swedish flag"><span>Swedish</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/ukrainian/" aria-label="Open Ukrainian benchmark"><img src="{{ base_url }}/assets/images/flags/uk.svg" alt="Ukrainian flag"><span>Ukrainian</span></a><a class="rx2-language" href="{{ base_url }}/benchmarks/languages/yiddish/" aria-label="Open Yiddish benchmark"><img src="{{ base_url }}/assets/images/flags/yi.svg" alt="Yiddish flag"><span>Yiddish</span></a></div>
|
||||
</section>
|
||||
|
||||
<section class="rx2-code-grid">
|
||||
<article class="rx2-code-card"><div class="rx2-code-title"><span class="java-symbol">♨</span><b>Java</b><button type="button" data-copy="java-code">Copy</button></div><pre id="java-code"><code><span class="kw">import</span> org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
<span class="kw">import</span> org.egothor.stemmer.FrequencyTrie;
|
||||
|
||||
<span class="type">FrequencyTrie</span><<span class="type">CompiledPatchCommand</span>> trie =
|
||||
StemmerPatchTrieLoader.loadCompiled(
|
||||
StemmerPatchTrieLoader.Language.US_UK, ...);
|
||||
|
||||
<span class="type">String</span> word = <span class="str">"running"</span>;
|
||||
<span class="type">CompiledPatchCommand</span> patch = trie.get(word);
|
||||
<span class="type">String</span> stem = patch == <span class="kw">null</span> ? word : patch.apply(word);</code></pre><a href="{{ base_url }}/quick-start/">View Java docs →</a></article>
|
||||
<article class="rx2-code-card"><div class="rx2-code-title"><span class="python-symbol">Py</span><b>Python</b><button type="button" data-copy="python-code">Copy</button></div><pre id="python-code"><code><span class="kw">from</span> radixor <span class="kw">import</span> Stemmer
|
||||
|
||||
stemmer = Stemmer(<span class="str">"en"</span>)
|
||||
stems = stemmer.stem_batch([
|
||||
<span class="str">"running"</span>, <span class="str">"studies"</span>, <span class="str">"better"</span>, <span class="str">"cars"</span>
|
||||
])</code></pre><a href="{{ base_url }}/python/quick-start/">View Python docs →</a></article>
|
||||
</section>
|
||||
|
||||
<section class="rx2-benefits">
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="m7 9 9-5 9 5v14l-9 5-9-5Z" fill="none" stroke="currentColor" stroke-width="2"/><path d="m7 9 9 5 9-5M16 14v14" fill="none" stroke="currentColor" stroke-width="2"/></svg><p><b>Learned transformations</b><span>Lexical evidence becomes<br>compact patch commands</span></p></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M5 24c4-9 9-13 22-16M8 9h7v7" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"/><path d="M4 27h24" stroke="currentColor" stroke-width="2.2" stroke-linecap="round"/></svg><p><b>Beyond the dictionary</b><span>Runtime is not a closed<br>word-to-lemma lookup</span></p></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M18 2 8 18h8l-2 12 10-17h-8Z" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linejoin="round"/></svg><p><b>Native speed</b><span>Multi-million words per<br>second in Python and Java</span></p></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><circle cx="16" cy="16" r="12" fill="none" stroke="currentColor" stroke-width="2"/><path d="M4 16h24M16 4c4 4 5 8 5 12s-1 8-5 12c-4-4-5-8-5-12s1-8 5-12Z" fill="none" stroke="currentColor" stroke-width="2"/></svg><p><b>20 language models</b><span>One architecture, trained<br>from language-specific data</span></p></div>
|
||||
<div><svg viewBox="0 0 32 32" aria-hidden="true"><path d="m16 3 10 4v7c0 8-4 12-10 15C10 26 6 22 6 14V7Z" fill="none" stroke="currentColor" stroke-width="2.1"/><path d="m11 15 3 3 7-7" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round"/></svg><p><b>Production ready</b><span>Deterministic behavior<br>and reproducible evidence</span></p></div>
|
||||
</section>
|
||||
|
||||
<section class="rx2-final-card">
|
||||
<img src="{{ base_url }}/assets/images/radixor-logo.png" alt="">
|
||||
<div><h2>A different stemming architecture.</h2><p>Radixor combines learned patch commands, trie-based structural sharing, subtree reduction, and deterministic runtime application. It is neither a flat dictionary lookup nor another fixed suffix-rule table.</p></div>
|
||||
<a class="rx2-gh" href="{{ base_url }}/technology-lineage/">Technology & lineage</a>
|
||||
<a class="rx2-doc" href="{{ base_url }}/getting-started/">▣ Read the Docs</a>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
{% endblock %}
|
||||
{% block footer %}<footer class="rx2-footer"><div class="rx2-shell"><span>© 2026 Egothor · Radixor software is BSD-3-Clause licensed.</span><span><a href="{{ base_url }}/technology-lineage/">Technology</a> · <a href="{{ base_url }}/compatibility-and-guarantees/">Compatibility</a> · <a href="{{ base_url }}/benchmarks/reference/reproducibility/">Reproducibility</a> · <a href="https://github.com/leogalambos/Radixor">GitHub</a></span></div></footer><script src="{{ base_url }}/assets/javascripts/landing-v2.js"></script>{% endblock %}
|
||||
@@ -1,7 +1,10 @@
|
||||
# Loading and Building Stemmers
|
||||
# Loading and Building Stemmers in Java
|
||||
|
||||
This document explains how to acquire a compiled Radixor stemmer in Java.
|
||||
|
||||
For Python construction and binary preparation, use [Python Usage and API](python/usage.md)
|
||||
and [Compiling Dictionaries in Python](python/model-compilation.md).
|
||||
|
||||
## Load a registered default model
|
||||
|
||||
Language-oriented entry points resolve a registered default model and compile its GZip textual dictionary into a `FrequencyTrie<CompiledPatchCommand>`. The corresponding model JAR must be on the runtime classpath; the core contains no dictionary.
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
# Programmatic Usage
|
||||
# Java Programmatic Usage
|
||||
|
||||
Radixor code and model data are separate runtime components. Every example on this page requires `org.egothor:radixor:<radixor-version>` as an `implementation` dependency and at least one model JAR as a runtime dependency. The core JAR contains no `stemmer.gz`.
|
||||
|
||||
The Python implementation has its own native API. `pip install radixor` also
|
||||
installs the separate standard data package containing 20 precompiled models.
|
||||
See the [Python Quick Start](python/quick-start.md) and
|
||||
[Python Usage and API](python/usage.md).
|
||||
|
||||
For complete dependency patterns, lifecycle guidance, and troubleshooting, use [Model Selection and Loading](model-selection-and-loading.md). The generated [model catalog](stemmer-model-catalog.md) records the current artifacts, versions, checksums, and provenance.
|
||||
|
||||
## 1. Minimal use: the Polish default
|
||||
|
||||
66
docs/python/fast-track.md
Normal file
66
docs/python/fast-track.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Python Fast Track
|
||||
|
||||
This is the shortest path from an empty Python environment to a working
|
||||
Radixor stemmer. The installation includes the native runtime and the separate
|
||||
standard-model package with 20 precompiled language models.
|
||||
|
||||
## 1. Install
|
||||
|
||||
=== "PyPI"
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: radixor
|
||||
```
|
||||
|
||||
=== "GitHub Releases"
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: \
|
||||
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
|
||||
```
|
||||
|
||||
PyPI publication is pending, and the GitHub index becomes live with the first
|
||||
Python releases. Until then, follow the source-checkout procedure on
|
||||
[Installation and Builds](installation.md).
|
||||
|
||||
Radixor supports CPython 3.9 and newer. A JVM, Java dependency, and source
|
||||
dictionary are not required.
|
||||
|
||||
## 2. Stem words
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
stemmer = Stemmer("en")
|
||||
|
||||
print(stemmer.stem("running"))
|
||||
print(stemmer.stem_batch(["running", "studies", "cars"]))
|
||||
```
|
||||
|
||||
Expected first output:
|
||||
|
||||
```text
|
||||
run
|
||||
```
|
||||
|
||||
`stem()` and `stem_batch()` preserve Radixor's original API: a word for which
|
||||
the trie finds no patch command produces `None`.
|
||||
|
||||
## 3. Use PyStemmer-compatible fallback semantics
|
||||
|
||||
For a low-friction migration from PyStemmer, use the compatible method names:
|
||||
|
||||
```python
|
||||
stemmer.stemWord("running")
|
||||
stemmer.stemWords(["running", "unknown_word"])
|
||||
```
|
||||
|
||||
These methods return the original input whenever no patch command is found, so
|
||||
their results are always strings rather than `None`.
|
||||
|
||||
## Next
|
||||
|
||||
- Continue with the [Python Quick Start](quick-start.md) for model selection,
|
||||
batch processing, custom compiled models, and deployment guidance.
|
||||
- Use [Python Usage and API](usage.md) as the method reference.
|
||||
- Review the reproducible [Python performance results](performance.md).
|
||||
96
docs/python/index.md
Normal file
96
docs/python/index.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Radixor for Python
|
||||
|
||||
The **`radixor`** package is Radixor's native Python implementation. It is not
|
||||
a wrapper around the Java library and does not require a JVM: it is a
|
||||
compiled extension (Rust, via [PyO3](https://pyo3.rs/) and
|
||||
[maturin](https://www.maturin.rs/)) that loads precompiled patch-command tries
|
||||
derived from the same canonical UniMorph data as the Java models.
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
s = Stemmer("en")
|
||||
s.stem("running") # 'run'
|
||||
s.stem_batch(["cats", "ran"]) # ['cat', 'run']
|
||||
```
|
||||
|
||||
- [Fast Track](fast-track.md) — install and produce the first stem.
|
||||
- [Quick Start](quick-start.md) — the complete application-oriented learning path.
|
||||
- [Installation and building](installation.md) — Linux, Windows, macOS.
|
||||
- [Usage and examples](usage.md) — batch API, caching, and custom models.
|
||||
- [Dictionary compilation](model-compilation.md) — prepare a version 7 binary
|
||||
once and share it with Python or Java.
|
||||
- [Performance](performance.md) — fair, reproducible comparisons vs PyStemmer,
|
||||
snowballstemmer, NLTK Porter, and CISTEM.
|
||||
|
||||
The language and model-ID mapping is shared with Java and maintained on the
|
||||
[Built-in Languages](../built-in-languages.md) page. Installing `radixor`
|
||||
also resolves the separate pure `radixor-models-standard` distribution containing
|
||||
the 20 default compiled models; Java applications select independently
|
||||
versioned model JARs.
|
||||
|
||||
!!! note "Same models, same results, different runtime"
|
||||
The standard Python models are compiled from the identical canonical
|
||||
dictionaries with the identical production reduction configuration
|
||||
(`MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS`, 75 % / 3×,
|
||||
uniform-subtree contraction, `LOWERCASE_WITH_LOCALE_ROOT`, `AS_IS`
|
||||
diacritics, `storeOriginal=true`). For a word present in a model, both
|
||||
implementations return the same dominant stem. The compiled **binary format
|
||||
is shared** (see below), so a model compiled by one side loads in the other.
|
||||
|
||||
## Java vs. Python: read this first
|
||||
|
||||
The two implementations solve the same problem but make different runtime
|
||||
trade-offs. Mixing their mental models causes confusion, so the differences are
|
||||
stated explicitly. **Neither is “better”** — they target different runtimes.
|
||||
|
||||
| Aspect | Java (`org.egothor:radixor`) | Python (`radixor`) |
|
||||
|---|---|---|
|
||||
| Runtime | JVM library | Compiled extension (Rust/PyO3), no JVM |
|
||||
| Distribution | Maven JAR + model JARs | `abi3` wheel (one wheel per OS/arch, Python ≥ 3.9) |
|
||||
| Hot-path data structure | `CompiledNode` graph; routines operate on caller-owned **`char[]`** with zero-copy normalized lookups and visitor sinks (`EntrySink`) | Flat **CSR arrays** (no per-node objects); reused UTF‑16 scratch buffers |
|
||||
| Result cache | **None** — `get()` is stateless and re-stems every call | **Bounded**, 10,000 entries by default (matching PyStemmer); `Stemmer(cache_size=0)` disables it |
|
||||
| Batch API | Not a batch call; you loop and reuse `char[]`/visitors to avoid allocation | **`stem_batch()` / `stem_all_batch()`** — one Python↔Rust crossing amortized over the whole list |
|
||||
| Reduction modes | All three modes selectable at compile time | Fixed to the production `DOMINANT` mode |
|
||||
| Extending a compiled trie | **Supported** — add words/transformations to an already-compiled trie without recompiling | **Not exposed** — compile from a dictionary (or load a compiled binary) |
|
||||
| Model resolution | `ServiceLoader` registry, descriptors, SHA‑256 integrity checks | Separate standard data package; catalog/format/SHA‑256 validation before synchronous native load |
|
||||
| Normalization control | Case and diacritic modes fully configurable | `lowercase` toggle; diacritics `AS_IS` (models are built this way) |
|
||||
| Binary format | `StemmerPatchTrieBinaryIO` v7 read/write (versioned, fingerprinted) | v7 read/write, **inner stream byte-identical** to Java; **v7 only** (no legacy v1–v6) |
|
||||
| Multiple stems | `getAll(...)` | `stem_all()` / `stem_all_batch()` |
|
||||
|
||||
### Runtime capabilities that differ
|
||||
|
||||
To avoid surprises, these Java capabilities are **not** in the Python package:
|
||||
|
||||
- **Extending / incrementally growing a compiled trie.** Python compiles from a
|
||||
source dictionary (or loads a compiled binary); it does not add words to an
|
||||
existing compiled trie at runtime.
|
||||
- **Selectable reduction modes.** Only the production `DOMINANT` mode is used.
|
||||
- **Pluggable provider discovery.** Python currently resolves one known
|
||||
standard provider directly; entry-point plugins are not yet exposed.
|
||||
- **Legacy binary versions.** Only stream version 7 is read/written.
|
||||
- **Diacritic-removal modes** beyond `AS_IS` (the bundled models are `AS_IS`).
|
||||
|
||||
### Python-specific capabilities
|
||||
|
||||
- A **batch API** (`stem_batch`) that amortizes the Python↔native boundary — the
|
||||
single most important call for throughput from Python.
|
||||
- A **bounded result cache** (`cache_size=10_000` by default) for workloads with
|
||||
repeated tokens. It is shared by `stem()`, `stemWord()`, `stem_batch()`, and
|
||||
`stemWords()`; pass `cache_size=0` to disable it. The `stem_all*()` methods are
|
||||
not cached.
|
||||
- A `lowercase=False` mode to skip per-lookup lowercasing when the caller
|
||||
guarantees already-lowercased input.
|
||||
|
||||
## Interoperability
|
||||
|
||||
The compiled binary is Radixor's **v7 trie stream**, and the Python runtime writes
|
||||
the *inner stream byte-for-byte identically to the Java*
|
||||
`StemmerPatchTrieBinaryIO`. Consequently:
|
||||
|
||||
- a model compiled by **Java** (`org.egothor.stemmer.Compile` /
|
||||
`StemmerPatchTrieBinaryIO.write`) loads in **Python**, and
|
||||
- a model compiled by **Python** (`radixor.compile(...)`) loads in **Java**.
|
||||
|
||||
(The outer gzip wrapper bytes differ between the two gzip implementations; this
|
||||
is irrelevant — both sides decompress to the same v7 stream.)
|
||||
215
docs/python/installation.md
Normal file
215
docs/python/installation.md
Normal file
@@ -0,0 +1,215 @@
|
||||
# Installing and building (Linux, Windows, macOS)
|
||||
|
||||
The Python package ships as an **`abi3` wheel** — a single binary per
|
||||
OS/architecture that works on CPython ≥ 3.9 (including 3.14) through the stable
|
||||
ABI. Most users just `pip install`; building from source is only needed for
|
||||
development or unsupported platforms.
|
||||
|
||||
## Install from PyPI
|
||||
|
||||
PyPI is the intended primary index once the Radixor projects are approved and
|
||||
published there:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: radixor
|
||||
```
|
||||
|
||||
PyPI publication is not live yet. Until the `radixor` and
|
||||
`radixor-models-standard` project pages exist, this command cannot install the
|
||||
project.
|
||||
|
||||
## Install compiled packages from GitHub
|
||||
|
||||
Python releases are published as immutable GitHub Release assets. A small
|
||||
PEP 503 index on GitHub Pages exposes both packages to `pip`:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: \
|
||||
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
|
||||
```
|
||||
|
||||
The index links directly to checksummed wheel assets in GitHub Releases; Pages
|
||||
does not duplicate the package files. It is not live until the first Python
|
||||
model and native releases have been published. This was verified before the
|
||||
initial release: the URL returned HTTP 404 and the repository contained no
|
||||
Python Release assets.
|
||||
|
||||
Do not configure the GitHub index as an `--extra-index-url`: `pip` does not
|
||||
prioritize one index over another. Use it as the sole `--index-url`, as shown
|
||||
above. The binary-only constraint also prevents an accidental source build
|
||||
with an unprepared toolchain.
|
||||
|
||||
Wheels are provided for Linux (`manylinux`), Windows, and macOS
|
||||
(x86‑64 and Apple Silicon). A source distribution is also published; installing
|
||||
it triggers a source build, which needs the toolchain described below.
|
||||
|
||||
## Install or build from the GitHub source repository
|
||||
|
||||
Building requires the **Rust toolchain**, a linker for the target platform, and
|
||||
**maturin**. The crate and its dependencies contain no project C/C++ sources,
|
||||
but the selected Rust target still needs its normal platform linker and SDK.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/leogalambos/Radixor
|
||||
cd Radixor
|
||||
python -m venv python/.venv
|
||||
# activate the venv (see per-OS note below)
|
||||
pip install maturin build setuptools wheel pytest
|
||||
./gradlew pythonBuildStandardModels
|
||||
pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
|
||||
cd python
|
||||
maturin develop --release # compile + install into the venv
|
||||
pytest -q # run the test suite
|
||||
```
|
||||
|
||||
For a reproducible application build, check out a release tag or exact commit
|
||||
instead of a moving branch. Repository descriptors deliberately use the
|
||||
non-release placeholder `0.0.0`; release workflows inject the version from the
|
||||
Git tag into isolated staging trees. Consequently, source-checkout development
|
||||
installs use `--no-deps` for the generated development model wheel, while
|
||||
published packages carry normal release versions and dependency resolution
|
||||
works automatically.
|
||||
|
||||
The native distribution requires
|
||||
`radixor-models-standard>=1.0,<2.0`; an installation of `radixor`
|
||||
resolves it automatically. The local `--no-deps` command installs the generated
|
||||
data wheel for development without contacting a package index.
|
||||
|
||||
The installed package source (package index, environment, and `sys.path`) is
|
||||
the model-provider trust boundary. Manifest SHA-256 checks detect accidental
|
||||
corruption after installation; they do not authenticate a malicious provider.
|
||||
|
||||
## Integrity and provenance
|
||||
|
||||
Every GitHub Release contains `SHA256SUMS` for its wheel and source archives.
|
||||
The release workflows also create GitHub artifact attestations for those
|
||||
archives. After downloading a release, maintainers and users can verify it with:
|
||||
|
||||
```bash
|
||||
sha256sum --check SHA256SUMS
|
||||
gh attestation verify radixor-<version>-<wheel-tags>.whl \
|
||||
--repo leogalambos/Radixor
|
||||
```
|
||||
|
||||
Python packages do **not** reuse the OpenPGP key configured for Java/Maven
|
||||
Central publications. Java's `SIGNING_KEY` and `SIGNING_PASSWORD` produce Maven
|
||||
signatures; Python currently uses release checksums plus GitHub's
|
||||
identity-bound build-provenance attestation. A future PyPI publication should
|
||||
use PyPI Trusted Publishing and its supported attestations rather than copying
|
||||
the Java signing mechanism.
|
||||
|
||||
## Build through Gradle
|
||||
|
||||
From the repository root, the supported build entry point creates the native
|
||||
wheel/sdist and pure standard-model wheel/sdist:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBuild
|
||||
```
|
||||
|
||||
Artifacts are written below `build/python/dist/`; they are not installed into
|
||||
the invoking interpreter. `./gradlew pythonVerifyDistributions` also validates
|
||||
archive contents, dependency metadata, checksums, v7 headers, and a fresh
|
||||
offline wheel-only installation. Platform convenience tasks are also available:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBuildLinux
|
||||
./gradlew pythonBuildWindows
|
||||
./gradlew pythonBuildMacos
|
||||
```
|
||||
|
||||
The task matching the current host delegates to `pythonBuild`. A non-host task
|
||||
uses the default Rust target for that operating system and therefore succeeds
|
||||
only when its Rust target, linker, and platform SDK are installed. Override a
|
||||
default with `pythonLinuxTarget`, `pythonWindowsTarget`, or `pythonMacosTarget`.
|
||||
For example:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBuildWindows -PpythonWindowsTarget=x86_64-pc-windows-gnu
|
||||
```
|
||||
|
||||
Use `-PpythonExecutable=/path/to/python` or
|
||||
`-PmaturinExecutable=/path/to/maturin` when those tools are not on `PATH`.
|
||||
These Gradle tasks are the repository integration; direct `maturin` commands
|
||||
remain useful while developing inside `python/`.
|
||||
|
||||
### Prerequisites per platform
|
||||
|
||||
=== "Linux"
|
||||
|
||||
```bash
|
||||
# Rust (rustup); most distros already ship Python 3.9+
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install maturin pytest
|
||||
```
|
||||
Producing distributable `manylinux` wheels is easiest with
|
||||
`maturin build --release` inside the official maturin/`manylinux` container.
|
||||
|
||||
=== "macOS"
|
||||
|
||||
```bash
|
||||
brew install rustup-init && rustup-init -y # or: curl https://sh.rustup.rs | sh
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install maturin pytest
|
||||
maturin develop --release
|
||||
```
|
||||
Both Apple Silicon (`aarch64-apple-darwin`) and Intel
|
||||
(`x86_64-apple-darwin`) are supported; `maturin build --release --target
|
||||
universal2-apple-darwin` produces a universal wheel.
|
||||
|
||||
=== "Windows"
|
||||
|
||||
```powershell
|
||||
winget install -e --id Rustlang.Rustup
|
||||
py -m venv .venv
|
||||
.\.venv\Scripts\Activate.ps1
|
||||
pip install maturin pytest
|
||||
maturin develop --release
|
||||
```
|
||||
The self-contained GNU toolchain avoids needing Visual Studio Build Tools:
|
||||
```powershell
|
||||
rustup toolchain install stable-x86_64-pc-windows-gnu
|
||||
rustup default stable-x86_64-pc-windows-gnu
|
||||
```
|
||||
(The MSVC toolchain also works if you already have the C++ Build Tools.)
|
||||
|
||||
### Python 3.14 (and newer than your PyO3 knows about)
|
||||
|
||||
Because the extension targets the stable ABI, it links against interpreters
|
||||
newer than the PyO3 version was released for. If a build against a very new
|
||||
CPython refuses, set the forward-compatibility flag once in the build shell:
|
||||
|
||||
=== "Linux / macOS"
|
||||
|
||||
```bash
|
||||
export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
|
||||
maturin develop --release
|
||||
```
|
||||
|
||||
=== "Windows (PowerShell)"
|
||||
|
||||
```powershell
|
||||
$env:PYO3_USE_ABI3_FORWARD_COMPATIBILITY = "1"
|
||||
maturin develop --release
|
||||
```
|
||||
|
||||
## Verifying the build
|
||||
|
||||
```bash
|
||||
python -c "from radixor import Stemmer; print(Stemmer('en').stem('running'))" # -> run
|
||||
pytest -q
|
||||
```
|
||||
|
||||
## Notes and caveats
|
||||
|
||||
- **Model packaging.** Neither runtime distribution contains textual
|
||||
dictionaries. `radixor-models-standard` ships 20 compiled gzip v7 resources,
|
||||
the checksum/provenance manifest, and CC BY-SA 3.0 notices; optional
|
||||
`pl-pl-polimorf` is excluded.
|
||||
- **Catalog compatibility.** Radixor 4.1 accepts model-distribution major 1
|
||||
(`>=1.0,<2.0`) carrying the independent 2026.1 catalog identity. Missing,
|
||||
incompatible, or corrupt data produces an
|
||||
actionable error before native loading.
|
||||
- **Toolchain PATH.** After installing rustup, open a fresh shell (or ensure
|
||||
`~/.cargo/bin` is on `PATH`) so `maturin` can find `cargo`/`rustc`.
|
||||
105
docs/python/model-compilation.md
Normal file
105
docs/python/model-compilation.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# Compiling Dictionaries in Python
|
||||
|
||||
The Python package can compile a textual Radixor dictionary into the shared
|
||||
version 7 binary trie format. This moves dictionary parsing, patch-command
|
||||
generation, trie construction, reduction, and serialization out of application
|
||||
startup.
|
||||
|
||||
Use this workflow when the application owns its model file. Standard language
|
||||
aliases already load validated, precompiled `.rxc` resources from
|
||||
`radixor-models-standard`; they do not parse or compile textual dictionaries
|
||||
when a `Stemmer` is constructed.
|
||||
|
||||
## Source format
|
||||
|
||||
The input is a plain UTF-8 or GZip-compressed UTF-8 tab-separated dictionary.
|
||||
The first column is the canonical stem and the remaining columns are its known
|
||||
surface forms:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
cat cats
|
||||
```
|
||||
|
||||
Remarks beginning with `#` or `//` are accepted. The complete syntax and
|
||||
normalization rules are documented on the shared [Dictionary Format](../dictionary-format.md)
|
||||
page.
|
||||
|
||||
## Compile a model
|
||||
|
||||
```python
|
||||
import radixor
|
||||
|
||||
radixor.compile(
|
||||
"stemmer.tsv.gz",
|
||||
"english.rxc",
|
||||
language="en",
|
||||
)
|
||||
```
|
||||
|
||||
`language` is used to choose traversal direction when `backward` is omitted.
|
||||
Persian (`fa`), Hebrew (`he`), and Yiddish (`yi`) use forward traversal; the
|
||||
other bundled languages use backward traversal. For a custom language, select
|
||||
the direction explicitly:
|
||||
|
||||
```python
|
||||
radixor.compile(
|
||||
"custom.tsv",
|
||||
"custom.rxc",
|
||||
backward=True,
|
||||
store_original=True,
|
||||
lowercase=True,
|
||||
)
|
||||
```
|
||||
|
||||
The arguments are:
|
||||
|
||||
| Argument | Meaning |
|
||||
|---|---|
|
||||
| `source` | Plain or GZip-compressed textual dictionary. |
|
||||
| `out_path` | Destination for the GZip-compressed version 7 trie. |
|
||||
| `language` | Optional alias or model ID used only to infer traversal direction. |
|
||||
| `backward` | Explicit traversal direction; overrides inference from `language`. |
|
||||
| `store_original` | Include a no-op mapping for every canonical stem. Defaults to `True`. |
|
||||
| `lowercase` | Record lowercase lookup normalization in the compiled metadata. Defaults to `True`. |
|
||||
|
||||
Compilation refuses an input that is already a compiled trie. The destination
|
||||
is written by the native extension; the caller is responsible for choosing its
|
||||
location and for replacing an existing file only when that is intended.
|
||||
|
||||
## Load the compiled model
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
stemmer = Stemmer(compiled="english.rxc")
|
||||
print(stemmer.stem("running"))
|
||||
```
|
||||
|
||||
`Stemmer(path=...)` also auto-detects textual dictionaries and compiled version
|
||||
7 streams, but `compiled=` communicates the deployment intent more clearly.
|
||||
Traversal direction, `store_original`, and lookup normalization are already
|
||||
stored in a compiled artifact; constructor build options do not rewrite them.
|
||||
|
||||
## Java interoperability
|
||||
|
||||
Python and Java share the inner version 7 trie stream. A binary produced by
|
||||
`radixor.compile(...)` can be loaded by Java's
|
||||
`StemmerPatchTrieLoader.loadBinaryCompiled(...)`, and Python can load a version
|
||||
7 artifact written by `StemmerPatchTrieBinaryIO`.
|
||||
|
||||
The outer GZip bytes need not be identical because compressor implementations
|
||||
may differ. Interoperability applies to the decompressed version 7 stream and
|
||||
its persisted metadata.
|
||||
|
||||
## Differences from the Java compiler
|
||||
|
||||
Python compilation intentionally exposes the production dominant-result
|
||||
configuration used by the Python runtime. Java additionally offers three selectable
|
||||
reduction modes, more normalization controls, incremental extension, and a CLI
|
||||
with explicit overwrite handling. Use [Java CLI Compilation](../cli-compilation.md)
|
||||
when those controls are required.
|
||||
|
||||
For normal Python use, compile once during preparation, deploy the resulting
|
||||
`.rxc` file as an application-owned asset, and reuse one loaded `Stemmer` at
|
||||
runtime.
|
||||
160
docs/python/performance.md
Normal file
160
docs/python/performance.md
Normal file
@@ -0,0 +1,160 @@
|
||||
# Performance (Python)
|
||||
|
||||
This page reports **runtime stemming throughput** of the Python implementation against
|
||||
common Python stemmers, and — crucially — documents exactly how the comparison
|
||||
is made fair. The scripts are in the repository (`python/benchmarks/`); anyone
|
||||
can reproduce the numbers.
|
||||
|
||||
!!! info "Published single-machine measurement"
|
||||
These results were regenerated on 2026-08-08 on the current benchmark
|
||||
workstation: Fedora Linux 44 (`7.1.6-201.fc44.x86_64`), AMD Ryzen 5 5625U
|
||||
(6 cores / 12 threads), CPython 3.14.6, Rust 1.97.1, and a release wheel.
|
||||
All logical CPUs used the `schedutil` governor. Absolute timings remain
|
||||
machine-specific; compare ratios only within this run.
|
||||
|
||||
## What is measured
|
||||
|
||||
- **Runtime stemming only.** Model construction / dictionary compilation happens
|
||||
once in setup and is excluded from every timing.
|
||||
- **Workload = the Java JMH corpus.** The *changed-token* corpus derived from
|
||||
the bundled UniMorph gold-standard dictionaries: every dictionary field paired
|
||||
with its line's root, normalized `trim().lower()`, keeping only tokens that
|
||||
differ from their root (the forms a stemmer must actually rewrite), padded to
|
||||
≥ 5 000 tokens. This is identical to the Java `LanguageBenchmarkCorpus`.
|
||||
- **Batch sizes 10/20/50/100** are swept and a line is fit to `per_call(N) =
|
||||
intercept + N · slope` as a descriptive scaling summary. This is an
|
||||
unconstrained OLS fit, so noise may produce a negative intercept; it is not a
|
||||
physical decomposition of runtime. The *best* of many repeats is reported.
|
||||
|
||||
## Fairness: making the comparison apples-to-apples
|
||||
|
||||
Three asymmetries silently distort stemmer comparisons. Each is neutralized, and
|
||||
where it **cannot** be neutralized the effect is described.
|
||||
|
||||
1. **Result caching — neutralized.** PyStemmer caches results by default
|
||||
(`maxCacheSize=10000`). Since a benchmark stems the same corpus repeatedly,
|
||||
that cache would turn measured passes into dictionary lookups rather than
|
||||
stemming. The harness explicitly disables **both** PyStemmer's cache
|
||||
(`maxCacheSize=0`) and radixor's default cache (`cache_size=0`). The other
|
||||
engines have no cache.
|
||||
2. **Lowercasing — neutralized.** Snowball (PyStemmer, snowballstemmer) and
|
||||
CISTEM differ in whether they case-fold. Snowball does **no** case handling;
|
||||
it assumes pre-lowercased input. The corpus is pre-lowercased for every
|
||||
engine, and radixor is therefore run with **`lowercase=False`** so it does
|
||||
the same work. On already-lowercased input radixor returns identical results
|
||||
either way. **Exception — CISTEM:** it always performs its own lowercasing
|
||||
and German umlaut normalization internally and cannot be told to skip it, so
|
||||
CISTEM does *slightly more* normalization work than the others. This
|
||||
unavoidable extra work biases the comparison modestly **in radixor's
|
||||
favour**, not CISTEM's.
|
||||
3. **Hidden delegation — neutralized.** `snowballstemmer` delegates to PyStemmer
|
||||
when PyStemmer is installed (they become the same C code). The harness
|
||||
bypasses that and uses snowballstemmer's genuine pure-Python backend, and
|
||||
records each engine's backing module + whether it is a compiled extension so
|
||||
the provenance is verifiable.
|
||||
|
||||
## Environment and parameters
|
||||
|
||||
| Item | Published value |
|
||||
|---|---|
|
||||
| CPU | AMD Ryzen 5 5625U with Radeon Graphics |
|
||||
| CPU topology | 6 physical cores / 12 logical CPUs |
|
||||
| OS | Fedora Linux 44, kernel `7.1.6-201.fc44.x86_64` |
|
||||
| CPU governor | `schedutil` on all 12 logical CPUs; boost enabled |
|
||||
| Python | CPython 3.14.6 |
|
||||
| Radixor | 4.1.0, release-mode ABI3 wheel, cache disabled |
|
||||
| PyStemmer | 3.1.0 (`libstemmer_c` 3.1.0), cache disabled |
|
||||
| snowballstemmer | 3.1.1, forced pure-Python backend |
|
||||
| NLTK | 3.10.2 |
|
||||
| Workload | 5,000 changed tokens per language and measurement |
|
||||
| Batch sizes | 10, 20, 50, 100 |
|
||||
| Timing | best of 15 measured passes after 3 warm-up passes |
|
||||
|
||||
The authoritative command was:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBenchmarkAllLanguagesBatch --rerun-tasks
|
||||
```
|
||||
|
||||
It completed successfully in 3 minutes 33 seconds and emitted
|
||||
the full per-size CSV and JSON reports under
|
||||
`build/reports/python-benchmarks/`.
|
||||
|
||||
## Results — batch size 100, cache disabled
|
||||
|
||||
The table reports nanoseconds per word at `N=100` (lower is better). A dash
|
||||
means that the engine has no implementation for that language. Every available
|
||||
competitor was measured in the same process, with the same corpus and batch
|
||||
partitioning.
|
||||
|
||||
| Language | Radixor | PyStemmer (Snowball C) | CISTEM (pure Py) | snowballstemmer (pure Py) | NLTK Porter (pure Py) |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| Czech (`cs`) | **224.3** | 236.6 | — | 4,835.2 | — |
|
||||
| Danish (`da`) | **178.3** | 267.6 | — | 8,568.9 | — |
|
||||
| German (`de`) | **230.9** | 635.5 | 3,341.9 | 33,654.1 | — |
|
||||
| English (`en`) | **180.5** | 331.9 | — | 20,195.0 | 7,740.3 |
|
||||
| Spanish (`es`) | **184.2** | 316.6 | — | 19,640.1 | — |
|
||||
| Persian (`fa`) | **210.1** | 497.1 | — | 32,732.3 | — |
|
||||
| Finnish (`fi`) | **227.8** | 258.8 | — | 12,339.5 | — |
|
||||
| French (`fr`) | **234.2** | 503.7 | — | 36,161.9 | — |
|
||||
| Hebrew (`he`) | **228.6** | — | — | — | — |
|
||||
| Hungarian (`hu`) | **198.2** | 264.7 | — | 13,694.3 | — |
|
||||
| Italian (`it`) | **170.8** | 517.0 | — | 34,504.6 | — |
|
||||
| Norwegian Bokmål (`nb`) | **187.1** | 239.7 | — | 7,457.6 | — |
|
||||
| Dutch (`nl`) | **187.1** | 354.8 | — | 18,148.2 | — |
|
||||
| Norwegian Nynorsk (`nn`) | **168.7** | 231.2 | — | 7,489.4 | — |
|
||||
| Polish (`pl`) | **194.6** | 214.5 | — | 5,282.9 | — |
|
||||
| Portuguese (`pt`) | **166.9** | 293.2 | — | 21,157.2 | — |
|
||||
| Russian (`ru`) | **273.4** | 414.4 | — | 15,703.8 | — |
|
||||
| Swedish (`sv`) | **189.3** | 212.5 | — | 5,351.4 | — |
|
||||
| Ukrainian (`uk`) | **221.5** | — | — | — | — |
|
||||
| Yiddish (`yi`) | **227.5** | 624.2 | — | 33,251.6 | — |
|
||||
|
||||
Radixor won all **18 / 18** direct PyStemmer comparisons. At `N=100`, its
|
||||
geometric-mean speedup was **1.67×**; the largest direct advantage was **3.03×**
|
||||
for Italian. Across all 20 Radixor languages, throughput ranged from **3.66 to
|
||||
5.99 million words/s**.
|
||||
|
||||
### CISTEM comparison for German
|
||||
|
||||
The German row also provides a direct comparison with CISTEM:
|
||||
|
||||
| Engine | Implementation | N=100 | vs radixor |
|
||||
|---|---|---|---|
|
||||
| **radixor** | Rust trie | **230.9 ns/word** | — |
|
||||
| PyStemmer (de) | Snowball C | 635.5 ns/word | 2.75× slower |
|
||||
| **CISTEM** | pure Python (`nltk`) | **3,341.9 ns/word** | **14.47× slower** |
|
||||
|
||||
CISTEM has no batch entry point (it is a per-word Python loop), so its per-word
|
||||
cost is flat across batch sizes and batching cannot amortize it. It is a compact
|
||||
~40-rule German heuristic with no dictionary — a different design point that
|
||||
trades coverage for simplicity. Because CISTEM's unavoidable normalization work
|
||||
modestly biases the measurement in radixor's favour (point 2 above), the 14.47×
|
||||
result is not a perfectly normalization-matched ratio.
|
||||
|
||||
The all-language Gradle task does not measure stage-level profiling or cached
|
||||
lookup performance. This page therefore does not mix such figures from an older
|
||||
workstation into the published run.
|
||||
|
||||
## A note on comparability of *quality*
|
||||
|
||||
These are **speed** comparisons. Radixor is a **lexicon-trained transformation
|
||||
stemmer**: it learns patch commands from UniMorph-grounded word–stem evidence
|
||||
and can generalize those transformations beyond exact training entries.
|
||||
Snowball, Porter, and CISTEM use hand-written rule systems. They produce
|
||||
different stems and are not directly comparable on output; see the shared
|
||||
[linguistic quality
|
||||
methodology](../benchmarks/reference/linguistic-quality.md) for how stemming
|
||||
quality is assessed separately from throughput.
|
||||
|
||||
## Reproduce
|
||||
|
||||
```bash
|
||||
pip install -r python/benchmarks/requirements-bench.txt
|
||||
./gradlew pythonBenchmarkAllLanguagesBatch --rerun-tasks
|
||||
```
|
||||
|
||||
The run prints the machine/Python/engine versions and each engine's backing
|
||||
module (provenance), and writes per-point rows (CSV) plus the full report
|
||||
including environment (JSON). Methodology and fairness notes live in
|
||||
`python/benchmarks/README.md`.
|
||||
130
docs/python/quick-start.md
Normal file
130
docs/python/quick-start.md
Normal file
@@ -0,0 +1,130 @@
|
||||
# Python Quick Start
|
||||
|
||||
Radixor's Python implementation is a native Rust extension with a Python API.
|
||||
It uses the same learned patch-command model and version 7 compiled-trie format
|
||||
as the Java implementation, without requiring a JVM.
|
||||
|
||||
## 1. Install the runtime and standard models
|
||||
|
||||
Create an isolated environment and install Radixor:
|
||||
|
||||
```bash
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
||||
```
|
||||
|
||||
=== "PyPI"
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: radixor
|
||||
```
|
||||
|
||||
=== "GitHub Releases"
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: \
|
||||
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
|
||||
```
|
||||
|
||||
PyPI publication is pending. The GitHub option becomes live when the first
|
||||
Python GitHub Releases populate the Pages-backed package index. See
|
||||
[Installation and Builds](installation.md) for availability and source builds.
|
||||
|
||||
The `radixor` wheel contains code. Its required
|
||||
`radixor-models-standard` dependency contains 20 precompiled models. The
|
||||
standard package excludes textual source dictionaries and optional PoliMorf
|
||||
data, which keeps startup on the direct compiled-model path.
|
||||
|
||||
## 2. Select and reuse a stemmer
|
||||
|
||||
Construct a stemmer once and retain it for the lifetime of the application:
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
english = Stemmer("en")
|
||||
polish = Stemmer("pl")
|
||||
|
||||
print(english.stem("running")) # 'run'
|
||||
print(polish.stem("koty"))
|
||||
```
|
||||
|
||||
Short aliases such as `en`, `de`, and `pl` resolve to the documented default
|
||||
model IDs. A full ID such as `us-uk-default` selects the same model explicitly.
|
||||
The complete mapping is listed under [Built-in Languages](../built-in-languages.md).
|
||||
|
||||
## 3. Prefer batch calls for collections
|
||||
|
||||
Crossing the Python/native boundary once per collection is substantially more
|
||||
efficient than a Python loop of scalar calls:
|
||||
|
||||
```python
|
||||
words = ["running", "studies", "better", "cars"]
|
||||
stems = english.stem_batch(words)
|
||||
```
|
||||
|
||||
`stem_batch()` preserves input order and returns one item per word. Entries can
|
||||
be `None` when the trie has no applicable patch command.
|
||||
|
||||
Repeated natural-language tokens use a bounded result cache shared by the
|
||||
scalar and batch APIs. Its default capacity is 10,000 entries, matching
|
||||
PyStemmer; choose another bound or pass `0` to disable it:
|
||||
|
||||
```python
|
||||
english = Stemmer("en", cache_size=10_000)
|
||||
uncached = Stemmer("en", cache_size=0)
|
||||
```
|
||||
|
||||
The cache covers `stem()`, `stemWord()`, `stem_batch()`, and `stemWords()`;
|
||||
the `stem_all*()` methods are not cached.
|
||||
|
||||
## 4. Migrate from PyStemmer
|
||||
|
||||
Radixor exposes PyStemmer's familiar scalar and batch method names:
|
||||
|
||||
```python
|
||||
stemmer = Stemmer("en")
|
||||
|
||||
stemmer.stemWord("running")
|
||||
stemmer.stemWords(["running", "unknown_word"])
|
||||
```
|
||||
|
||||
`stemWord()` and `stemWords()` return unmatched input unchanged. This removes
|
||||
the `None` fallback checks required by Radixor's original `stem()` and
|
||||
`stem_batch()` methods, so most migration work is limited to the package import
|
||||
and dependency change.
|
||||
|
||||
## 5. Load a custom compiled model
|
||||
|
||||
The standard installation covers the maintained default catalog. A custom
|
||||
version 7 model can be loaded directly:
|
||||
|
||||
```python
|
||||
custom = Stemmer(compiled="models/domain-english.rxc")
|
||||
```
|
||||
|
||||
To compile a maintained textual dictionary during a preparation step:
|
||||
|
||||
```python
|
||||
from radixor import compile
|
||||
|
||||
compile("dictionaries/domain.tsv.gz", "models/domain-english.rxc", language="en")
|
||||
```
|
||||
|
||||
Deploy the resulting `.rxc` file and load it at application startup. See
|
||||
[Dictionary Compilation](model-compilation.md) for format interoperability and
|
||||
the production compilation profile.
|
||||
|
||||
## 6. Production checklist
|
||||
|
||||
- Pin compatible `radixor` and `radixor-models-standard` releases in the
|
||||
application's dependency lock.
|
||||
- Construct and reuse stemmers instead of rebuilding them per request.
|
||||
- Use batch calls for token collections.
|
||||
- Choose `stem*` or `stemWord*` semantics deliberately for unmatched words.
|
||||
- Treat custom dictionaries and compiled models as trusted application input.
|
||||
- Regression-test representative vocabulary before changing model versions.
|
||||
|
||||
Continue with [Installation and Builds](installation.md) for wheel/platform
|
||||
details, [Usage and API](usage.md) for the complete call surface, or
|
||||
[Performance](performance.md) for benchmark methodology and results.
|
||||
173
docs/python/usage.md
Normal file
173
docs/python/usage.md
Normal file
@@ -0,0 +1,173 @@
|
||||
# Usage and examples
|
||||
|
||||
## Creating a stemmer
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
s = Stemmer("en") # by language code (bundled model)
|
||||
s = Stemmer("us-uk-default") # by full model ID
|
||||
s = Stemmer(path="my_dictionary.gz") # a custom gzipped TSV source dictionary
|
||||
s = Stemmer(compiled="en.rxc") # a pre-compiled binary (instant load)
|
||||
```
|
||||
|
||||
The traversal direction is derived from the language (right-to-left `fa`/`he`/`yi`
|
||||
use FORWARD, all others BACKWARD); override with `backward=True|False` for a
|
||||
custom `path=`.
|
||||
|
||||
## Stemming a single word
|
||||
|
||||
```python
|
||||
s.stem("running") # 'run'
|
||||
s.stem("cats") # 'cat'
|
||||
s.stem("zzzzz") # None -> not reducible / unknown
|
||||
```
|
||||
|
||||
`stem()` returns the single **dominant** stem, or `None`.
|
||||
|
||||
!!! info "Why a known word may return itself"
|
||||
A surface form that is *also* a canonical headword (e.g. an English word
|
||||
that is both its own lemma and an inflection of another lemma) returns
|
||||
itself, because the dominant sense is “this word is its own stem”. The
|
||||
inflectional reading is still available via `stem_all()`.
|
||||
|
||||
## Batch stemming — the fast path
|
||||
|
||||
For anything beyond a handful of words, use the batch API. It crosses the
|
||||
Python↔native boundary **once** for the whole list, which is the dominant cost
|
||||
when stemming from Python.
|
||||
|
||||
```python
|
||||
words = ["running", "cats", "stemming", "quickly"]
|
||||
s.stem_batch(words) # ['run', 'cat', 'stem', 'quick'] (None for unknowns)
|
||||
```
|
||||
|
||||
```python
|
||||
# Multiple candidate stems per word (ambiguity preserved):
|
||||
s.stem_all("running") # e.g. ['run', 'runn']
|
||||
s.stem_all_batch(["running", "cats"])
|
||||
```
|
||||
|
||||
## PyStemmer-compatible methods
|
||||
|
||||
Radixor also exposes PyStemmer's scalar and batch method names. They differ
|
||||
from the native Radixor methods only when the trie cannot find a patch command:
|
||||
|
||||
| Method | Recognized word | Word without a patch command | Return type |
|
||||
| --- | --- | --- | --- |
|
||||
| `stem(word)` | dominant stem | `None` | `str | None` |
|
||||
| `stem_batch(words)` | dominant stem at the same position | `None` at the same position | `list[str | None]` |
|
||||
| `stemWord(word)` | dominant stem | original input word | `str` |
|
||||
| `stemWords(words)` | dominant stem at the same position | original input word at the same position | `list[str]` |
|
||||
|
||||
Use `stemWord()` and `stemWords()` when migrating code that expects
|
||||
PyStemmer's no-`None` contract:
|
||||
|
||||
```python
|
||||
import radixor as Stemmer
|
||||
|
||||
# The rest of this common PyStemmer call pattern remains unchanged.
|
||||
s = Stemmer.Stemmer("english")
|
||||
|
||||
s.stemWord("running") # 'run'
|
||||
s.stemWord("unknown_word") # 'unknown_word'
|
||||
s.stemWords(["running", "unknown_word"])
|
||||
# ['run', 'unknown_word']
|
||||
```
|
||||
|
||||
`stemWords()` retains input order and makes one Python-to-Rust call for the
|
||||
whole list. PyStemmer's full language names, such as `"english"` and
|
||||
`"czech"`, are accepted for bundled Radixor languages alongside two-letter
|
||||
codes and full model IDs.
|
||||
|
||||
The compatibility contract covers these method names, full language aliases,
|
||||
and unmatched-word fallback behavior. Radixor configuration keywords remain
|
||||
Radixor-specific: use `cache_size`, not PyStemmer's `maxCacheSize`. Both
|
||||
libraries default to a cache capacity of 10,000 entries.
|
||||
|
||||
## Bounded result cache
|
||||
|
||||
Real text repeats tokens. The default bounded cache returns the already-built
|
||||
result object on a recognized-word hit (a reference-count bump — no
|
||||
re-stemming, no new result string). Unknown words are cached as misses, so
|
||||
`stemWord()` and `stemWords()` still create their required original-word
|
||||
result. Its default capacity is **10,000 entries**, matching PyStemmer:
|
||||
|
||||
```python
|
||||
s = Stemmer("en") # cache up to 10,000 distinct input words
|
||||
s = Stemmer("en", cache_size=50_000) # choose a custom capacity
|
||||
s = Stemmer("en", cache_size=0) # explicitly disable caching
|
||||
```
|
||||
|
||||
One cache is shared by `stem()`, `stemWord()`, `stem_batch()`, and
|
||||
`stemWords()`. The `stem_all()` and `stem_all_batch()` methods are not cached.
|
||||
Caching never changes results; it only avoids recomputation. Entries are
|
||||
inserted until the configured capacity is reached; there is no eviction. For a
|
||||
high-cardinality stream without useful token repetition, use `cache_size=0`.
|
||||
|
||||
## Skipping lowercasing for pre-normalized input
|
||||
|
||||
By default lookups lowercase the input (`LOWERCASE_WITH_LOCALE_ROOT`). If your
|
||||
pipeline already lowercases tokens, skip the redundant work:
|
||||
|
||||
```python
|
||||
s = Stemmer("en", lowercase=False) # assume already-lowercased input
|
||||
s.stem("running") # 'run'
|
||||
s.stem("Running") # None -> not lowercased, so no match
|
||||
```
|
||||
|
||||
The model's keys are always lowercase; `lowercase=False` only turns off
|
||||
per-lookup normalization. On already-lowercased input the results are identical.
|
||||
|
||||
## Compile once, load instantly
|
||||
|
||||
Compiling a trie from text costs a few seconds for large languages. Compile it
|
||||
once to Radixor's binary format and load it directly afterwards:
|
||||
|
||||
```python
|
||||
import radixor
|
||||
|
||||
radixor.compile("stemmer.gz", "en.rxc", language="en")
|
||||
s = radixor.Stemmer(compiled="en.rxc")
|
||||
```
|
||||
|
||||
See [Compiling Dictionaries in Python](model-compilation.md) for the source
|
||||
format, traversal and normalization options, deployment guidance, Java
|
||||
interoperability, and the controls that remain Java-only.
|
||||
|
||||
## Using a custom dictionary
|
||||
|
||||
A source dictionary is a gzipped (or plain) TSV file, one entry per line, the
|
||||
first column the canonical stem and the rest its variants; `#` and `//` start
|
||||
line remarks:
|
||||
|
||||
```
|
||||
run running runs ran
|
||||
cat cats
|
||||
```
|
||||
|
||||
```python
|
||||
s = Stemmer(path="custom.gz", backward=True, store_original=True)
|
||||
```
|
||||
|
||||
`store_original=True` (default) maps each stem to itself (a no-op patch) so the
|
||||
stem is recognised. See [Dictionary Format](../dictionary-format.md) for the
|
||||
authoritative specification shared with the Java project.
|
||||
|
||||
## Thread-safety
|
||||
|
||||
A `Stemmer` is safe to share across threads. The bounded cache is guarded
|
||||
internally; the compiled trie is immutable after construction.
|
||||
|
||||
## API summary
|
||||
|
||||
| Call | Returns | Notes |
|
||||
|---|---|---|
|
||||
| `Stemmer(lang \| path= \| compiled=, *, backward, store_original, lowercase, cache_size=10_000)` | stemmer | auto-detects compiled vs textual for `path=`; `cache_size=0` disables caching |
|
||||
| `stem(word)` | `str \| None` | dominant stem |
|
||||
| `stem_batch(words)` | `list[str \| None]` | **preferred** for many words |
|
||||
| `stemWord(word)` | `str` | PyStemmer-compatible; returns an unmatched word unchanged |
|
||||
| `stemWords(words)` | `list[str]` | PyStemmer-compatible batch call; preserves unmatched words and input order |
|
||||
| `stem_all(word)` | `list[str]` | all candidate stems, best first |
|
||||
| `stem_all_batch(words)` | `list[list[str]]` | |
|
||||
| `radixor.compile(source, out, *, language, backward, store_original, lowercase)` | `None` | writes a v7 binary |
|
||||
@@ -1,6 +1,9 @@
|
||||
# Quick Start
|
||||
# Java Quick Start
|
||||
|
||||
This guide introduces the fastest practical path to using **Radixor**.
|
||||
This guide introduces the fastest practical path to using **Radixor in Java**.
|
||||
Python users have an equivalent application-oriented path in the
|
||||
[Python Quick Start](python/quick-start.md), followed by the
|
||||
[Python Usage and API guide](python/usage.md).
|
||||
|
||||
If you are new to Radixor and want the shortest possible path to a first working stem, start with
|
||||
[Fast Track](fast-track.md). This Quick Start is a broader developer walkthrough: it introduces the
|
||||
|
||||
236
docs/technology-lineage.md
Normal file
236
docs/technology-lineage.md
Normal file
@@ -0,0 +1,236 @@
|
||||
# Technology and Lineage
|
||||
|
||||
Radixor has an unusual position in the stemming ecosystem because its history
|
||||
predates several implementations with which it is now compared.
|
||||
|
||||
The useful way to describe that history is not as a list of project names, but
|
||||
as a set of **algorithmic lineages and runtime models**.
|
||||
|
||||
## The Egothor lineage
|
||||
|
||||
The transformation-based approach behind Radixor was described by Leo Galambos
|
||||
in *Lemmatizer for Document Information Retrieval Systems in JAVA* (SOFSEM
|
||||
2001). The Egothor implementation developed the patch-command/trie approach used
|
||||
to compile word-form transformations into a compact stemming structure.
|
||||
|
||||
Radixor is a modern implementation of that lineage. It is not a binary or source
|
||||
repackaging of the old project: the current implementation has a new runtime
|
||||
representation, compiled patch commands, deterministic multi-result semantics,
|
||||
modern reduction modes, persistence, model packaging, integrity validation, and
|
||||
current Java/Python integration.
|
||||
|
||||
See [Why Radixor Is Different](why-radixor-is-different.md) for the architecture
|
||||
rather than the chronology.
|
||||
|
||||
## Stempel is a historical Egothor branch, not an independent algorithmic lineage
|
||||
|
||||
This point is easy to miss when Stempel is encountered through Lucene or
|
||||
Elasticsearch.
|
||||
|
||||
Lucene's own Stempel documentation states that the core stemming algorithm and
|
||||
implementation were taken **verbatim / virtually unchanged from the Egothor
|
||||
project**. The Stempel distribution is principally associated with its Polish
|
||||
stemming tables, even though the underlying algorithm is not inherently
|
||||
Polish-specific.
|
||||
|
||||
That makes Stempel historically important, but it should be interpreted
|
||||
correctly in Radixor comparisons:
|
||||
|
||||
- it demonstrates that the older Egothor technique survived in major search
|
||||
infrastructure;
|
||||
- it is not evidence of a separate later algorithm that Radixor subsequently
|
||||
copied;
|
||||
- benchmarking Stempel against Radixor is effectively a comparison between a
|
||||
preserved legacy branch of the technique and its modern re-engineering.
|
||||
|
||||
Official Lucene reference:
|
||||
[StempelStemmer](https://lucene.apache.org/core/9_9_1/analysis/stempel/org/apache/lucene/analysis/stempel/StempelStemmer.html).
|
||||
|
||||
In the current Polish benchmark, the direct Stempel path has balanced accuracy
|
||||
**0.855699** versus **0.991105** for Radixor, and its direct runtime is measured
|
||||
at **4.229×** the Radixor time. The Lucene StempelFilter path is **4.803×** the
|
||||
Radixor time. See the [Polish benchmark](benchmarks/languages/polish.md).
|
||||
|
||||
## Morfologik is a closed-vocabulary morphological lookup
|
||||
|
||||
Morfologik solves a different problem from Radixor. Its runtime is a
|
||||
dictionary-driven morphological lookup backed by a finite-state automaton (FSA).
|
||||
The distinction is not merely terminology: it determines what happens when
|
||||
production text contains a word form that the dictionary does not know.
|
||||
|
||||
`DictionaryLookup.lookup(...)` searches the compiled automaton for the supplied
|
||||
surface form and returns the stored base-form analyses only when that form is
|
||||
present. If the lookup fails, it returns an empty result. Lucene's
|
||||
`MorfologikFilter` tries the original token and then its lowercase form; if both
|
||||
lookups fail, the filter emits the original token unchanged.
|
||||
|
||||
There is therefore **no rule-based or learned transformation fallback for an
|
||||
out-of-vocabulary word**.
|
||||
|
||||
Primary-source implementations:
|
||||
|
||||
- [Morfologik `DictionaryLookup`](https://github.com/morfologik/morfologik-stemming/blob/master/morfologik-stemming/src/main/java/morfologik/stemming/DictionaryLookup.java)
|
||||
- [Lucene `MorfologikFilter`](https://github.com/apache/lucene/blob/main/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java)
|
||||
|
||||
This makes dictionary completeness an operational requirement, not just a
|
||||
quality-tuning parameter. A new domain term, previously unseen inflection,
|
||||
product name, spelling variant, or other surface form outside the compiled
|
||||
dictionary receives no morphological reduction from Morfologik. In an
|
||||
open-vocabulary search system, maintaining coverage therefore requires a
|
||||
sufficiently comprehensive dictionary and continued dictionary updates.
|
||||
|
||||
Radixor uses lexical resources differently. Its source data is **training
|
||||
evidence for transformations**. Word-to-root relationships are converted into
|
||||
patch commands, organized in a trie, structurally reduced, and compiled into a
|
||||
runtime machine. The deployed stemmer selects transformation behaviour and
|
||||
applies it to the input token; it is not restricted to retrieving a stored
|
||||
analysis for an exact dictionary member.
|
||||
|
||||
This architectural difference matters when interpreting quality numbers.
|
||||
Morfologik can provide strong analyses for vocabulary covered by its dictionary,
|
||||
but that strength does not imply generalization to unseen forms. Radixor is
|
||||
designed to preserve the linguistic evidence of large lexical resources while
|
||||
turning it into reusable transformation behaviour.
|
||||
|
||||
The current Polish benchmark also places the two approaches at very different
|
||||
points on the measured quality/performance envelope:
|
||||
|
||||
- deterministic primary-output balanced accuracy is **0.991105** for Radixor
|
||||
and **0.948392** for `MorfologikFilter`;
|
||||
- when all emitted candidates are considered, balanced accuracy is
|
||||
**1.000000** for Radixor and **0.987528** for Morfologik;
|
||||
- the measured Lucene `MorfologikFilter` runtime is **15.997×** the Radixor
|
||||
runtime in the same Java benchmark.
|
||||
|
||||
See the [Polish benchmark](benchmarks/languages/polish.md).
|
||||
|
||||
The useful conclusion is not that Morfologik is unsophisticated. It is a
|
||||
morphological dictionary system with a richer analysis objective. The important
|
||||
engineering distinction is sharper: **it pays the runtime and storage cost of
|
||||
dictionary/FSA analysis while remaining bounded by dictionary coverage;
|
||||
Radixor compiles lexical evidence into a substantially smaller hot-path
|
||||
transformation problem that can also operate beyond explicitly observed word
|
||||
forms.**
|
||||
|
||||
## Snowball and Porter are fixed rule systems
|
||||
|
||||
Porter and Snowball form another distinct lineage. Their language algorithms are
|
||||
explicit rule programs, usually centered on suffix regions and ordered rewrite
|
||||
rules.
|
||||
|
||||
They have a genuine advantage over closed dictionary lookup: their rules
|
||||
naturally apply to previously unseen words. The trade-off is that the
|
||||
linguistic behaviour is encoded in the hand-designed rule program itself rather
|
||||
than learned from lexical evidence.
|
||||
|
||||
At `N=100`, the current Python batch benchmark shows that rule-based
|
||||
generalization does not require accepting a runtime advantage over Radixor:
|
||||
|
||||
- **18 / 18** direct language comparisons are won by Radixor against PyStemmer
|
||||
3.1.0;
|
||||
- the geometric-mean speedup is **1.67×**;
|
||||
- the largest measured direct advantage is **3.03×** (Italian);
|
||||
- Radixor spans **3.66–5.99 million words/s** across all 20 measured Radixor
|
||||
languages at batch size `N=100`.
|
||||
|
||||
Those are performance results. The newly integrated official Snowball 3.1.0 Java
|
||||
quality comparators also make the linguistic trade-off visible for the three
|
||||
algorithms added in that Snowball generation:
|
||||
|
||||
| Language, `ALL_WORDS` | Radixor balanced accuracy | Snowball 3.1.0 balanced accuracy | Radixor OI / UI | Snowball OI / UI |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Czech | **0.996617** | 0.786366 | **0% / 0.676519%** | 0.000904% / 42.725842% |
|
||||
| Persian | **0.976360** | 0.535123 | **0% / 4.728041%** | 0.001278% / 92.974054% |
|
||||
| Polish | **0.991105** | 0.823625 | **0% / 1.779024%** | 0.000967% / 35.273970% |
|
||||
|
||||
The lowercase-only evaluation gives the same picture:
|
||||
|
||||
- **Czech:** 0.997195 vs 0.784821 balanced accuracy, with UI 0.561033% vs
|
||||
43.034822%;
|
||||
- **Persian:** 0.976360 vs 0.535123, with UI 4.728041% vs 92.974054%;
|
||||
- **Polish:** 0.991301 vs 0.823465, with UI 1.739895% vs 35.306102%.
|
||||
|
||||
The dominant difference is under-stemming rather than excessive conflation.
|
||||
Snowball's over-stemming remains very low in these measurements, but it leaves
|
||||
a much larger share of gold-related forms ungrouped. That distinction matters:
|
||||
a conservative stemmer can look safe when judged only by false conflations
|
||||
while still sacrificing substantial recall.
|
||||
|
||||
Finnish remains another useful illustration. The published `ALL_WORDS`
|
||||
primary-output quality result is **0.984838** balanced accuracy for Radixor
|
||||
versus **0.740279** for the Snowball Finnish Lucene path, with under-stemming
|
||||
**3.032474%** versus **51.944179%**. At `N=100` in the current Python batch run,
|
||||
Radixor is **1.14×** faster than PyStemmer's Finnish implementation.
|
||||
|
||||
See the [Finnish benchmark](benchmarks/languages/finnish.md), the
|
||||
[Czech benchmark](benchmarks/languages/czech.md), the
|
||||
[Persian benchmark](benchmarks/languages/persian.md), and the
|
||||
[Polish benchmark](benchmarks/languages/polish.md).
|
||||
|
||||
The important comparison is therefore not “dictionary versus rules”.
|
||||
Radixor occupies a third position: **it learns transformations from lexical
|
||||
evidence, compiles them into a reduced patch-command trie, and retains
|
||||
algorithmic generalization at runtime without hardcoding a fixed suffix program.**
|
||||
|
||||
## Lucene light, minimal, plural, and possessive filters
|
||||
|
||||
Several Lucene language filters are intentionally narrow transformations. A
|
||||
minimal or light stemmer may be extremely fast precisely because it performs
|
||||
less linguistic conflation.
|
||||
|
||||
That is not a defect. It is a different objective.
|
||||
|
||||
The important benchmark discipline is therefore:
|
||||
|
||||
> do not interpret runtime without the corresponding grouping quality.
|
||||
|
||||
A stemmer that removes only a tiny set of endings and a stemmer that attempts
|
||||
broad morphological conflation are not doing equivalent work merely because
|
||||
both return a string called a “stem”.
|
||||
|
||||
## Hunspell
|
||||
|
||||
Hunspell combines dictionaries with affix rules and can produce several
|
||||
candidate stems. It is another useful comparator because it occupies a middle
|
||||
ground between direct dictionary analysis and pure suffix stemming.
|
||||
|
||||
Its architecture is still different from Radixor's compiled patch-command trie:
|
||||
Hunspell interprets lexical and affix resources, whereas Radixor has already
|
||||
compiled observed transformation behaviour into a reduced runtime machine.
|
||||
|
||||
## What the current benchmark results justify saying
|
||||
|
||||
The project does not need to position Radixor merely as “another stemmer”.
|
||||
|
||||
A more accurate statement is:
|
||||
|
||||
> **Radixor is a learned transformation stemmer built around reduced
|
||||
> patch-command tries. Its current public benchmarks show that this architecture
|
||||
> can move the quality/performance frontier rather than merely trade one for the
|
||||
> other.**
|
||||
|
||||
That is an architectural and empirical claim, not a claim that every alternative
|
||||
project is poorly designed. Different systems were built for different goals:
|
||||
|
||||
| Family | Primary runtime idea | Typical strength | Key distinction from Radixor |
|
||||
| --- | --- | --- | --- |
|
||||
| Radixor | Learned patch commands in a reduced compiled trie | High-quality conflation with compact deterministic runtime | Training data compiles into reusable transformations |
|
||||
| Stempel | Historical Egothor implementation + stemming tables | Proven legacy deployment, especially Polish | Same historical algorithmic lineage; older implementation branch |
|
||||
| Morfologik | Dictionary/FSA morphological lookup | Rich in-vocabulary lemmatization and multiple analyses | Closed-vocabulary lookup: unknown forms have no stemming fallback; Lucene passes them through unchanged |
|
||||
| Snowball / Porter | Fixed language rule programs | Portable rule-based stemming with natural OOV coverage | Rules generalize to unseen forms, but are authored rather than learned from lexical evidence |
|
||||
| Lucene light/minimal | Deliberately narrow handcrafted rules | Very low runtime cost | Intentionally less linguistic work |
|
||||
| Hunspell | Dictionary + affix rules | Lexical/affix analysis and candidate outputs | Runtime interprets lexicon/affix resources |
|
||||
|
||||
The benchmark pages remain the authority for each language and comparator. The
|
||||
purpose of this page is to make the **technology categories** explicit so readers
|
||||
do not have to infer them from implementation names.
|
||||
|
||||
## Historical references
|
||||
|
||||
- Leo Galambos, *Lemmatizer for Document Information Retrieval Systems in JAVA*
|
||||
(SOFSEM 2001)
|
||||
- [Lucene StempelStemmer documentation](https://lucene.apache.org/core/9_9_1/analysis/stempel/org/apache/lucene/analysis/stempel/StempelStemmer.html)
|
||||
- [Lucene Morfologik package documentation](https://lucene.apache.org/core/10_3_2/analysis/morfologik/org/apache/lucene/analysis/morfologik/package-summary.html)
|
||||
- [Architecture](architecture.md)
|
||||
- [Benchmark results](benchmarks/index.md)
|
||||
- [Tested stemmer inventory](benchmarks/reference/tested-stemmers.md)
|
||||
161
docs/why-radixor-is-different.md
Normal file
161
docs/why-radixor-is-different.md
Normal file
@@ -0,0 +1,161 @@
|
||||
# Why Radixor Is Different
|
||||
|
||||
Radixor is **dictionary-trained, not dictionary-bound**.
|
||||
|
||||
The source dictionaries are build-time evidence from which Radixor learns
|
||||
word-to-stem transformations. The runtime artifact is not a flat table that can
|
||||
only answer words already present in that evidence. It is a compact,
|
||||
deterministic **trie of patch commands**.
|
||||
|
||||
That distinction is the shortest way to understand the project.
|
||||
|
||||
## A learned transformation stemmer
|
||||
|
||||
A conventional dictionary lookup stores a relationship such as:
|
||||
|
||||
```text
|
||||
running -> run
|
||||
```
|
||||
|
||||
Radixor instead derives a transformation that can be represented conceptually as:
|
||||
|
||||
```text
|
||||
running -> <patch command> -> run
|
||||
```
|
||||
|
||||
Many word forms share the same transformation behaviour. Radixor organizes those
|
||||
commands in a trie, reduces structurally equivalent regions, contracts uniform
|
||||
preferred-command subtrees, and freezes the result into an immutable compiled
|
||||
runtime structure.
|
||||
|
||||
The pipeline is therefore:
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
evidence[Lexical evidence]
|
||||
transformations[Word-to-root transformations]
|
||||
commands[Patch commands]
|
||||
mutable[Mutable trie]
|
||||
reduction[Subtree contraction and semantic reduction]
|
||||
compiled[Compiled trie]
|
||||
runtime[Runtime command selection and application]
|
||||
|
||||
evidence --> transformations --> commands --> mutable
|
||||
mutable --> reduction --> compiled --> runtime
|
||||
```
|
||||
|
||||
The dictionary is important because it supplies the linguistic evidence. It does
|
||||
**not** define a closed runtime vocabulary.
|
||||
|
||||
## What happens to an unseen word?
|
||||
|
||||
The compiled trie selects transformation behaviour rather than storing a full
|
||||
lemma string for every possible input.
|
||||
|
||||
Uniform subtrees can be contracted into accepting leaves. When lookup reaches an
|
||||
accepting leaf whose preferred patch command is already determined, the runtime
|
||||
can apply that command even with input characters remaining. This is one of the
|
||||
ways the compiled model can generalize beyond explicitly observed dictionary
|
||||
forms.
|
||||
|
||||
Generalization is not a promise that every arbitrary unknown token has a useful
|
||||
stem. No practical stemmer can make that guarantee. The important property is
|
||||
that Radixor is **not limited to exact dictionary membership**.
|
||||
|
||||
For the implementation details, see [Architecture](architecture.md).
|
||||
|
||||
## Why patch commands matter
|
||||
|
||||
Patch commands encode *how to transform* a word rather than merely *which string
|
||||
to return*.
|
||||
|
||||
That gives the runtime model several useful properties:
|
||||
|
||||
- repeated transformation behaviour can be shared;
|
||||
- trie paths share structural information between related inputs;
|
||||
- equivalent subtrees can be reduced;
|
||||
- the final command is applied directly to the original token;
|
||||
- the runtime can expose a deterministic preferred result;
|
||||
- the same compiled node may retain ranked alternative commands when ambiguity
|
||||
should not be discarded.
|
||||
|
||||
The result is closer to a compact learned transformation machine than to either a
|
||||
flat dictionary or a handwritten suffix list.
|
||||
|
||||
## Why the trie matters
|
||||
|
||||
The trie is not just a storage container around a dictionary.
|
||||
|
||||
It is the structure that makes the learned transformations reusable. Shared
|
||||
paths represent shared input structure; reduction merges equivalent behaviour;
|
||||
uniform-subtree contraction can terminate preferred-result lookup early.
|
||||
|
||||
This is also why the source dictionary may be very large while the deployed
|
||||
representation remains compact and fast.
|
||||
|
||||
## Radixor is not three common things
|
||||
|
||||
### It is not a closed dictionary lemmatizer
|
||||
|
||||
A closed dictionary lemmatizer primarily asks whether the current surface form
|
||||
exists in a lexicon or automaton and, if it does, returns stored analyses.
|
||||
|
||||
Radixor uses lexical resources differently: it **compiles transformation
|
||||
behaviour from them**.
|
||||
|
||||
### It is not another fixed suffix-rule stemmer
|
||||
|
||||
Porter- and Snowball-family stemmers encode explicit rules for a language.
|
||||
Those systems can generalize because the rules apply to unseen text, but the
|
||||
rules themselves are fixed algorithmic knowledge.
|
||||
|
||||
Radixor learns its transformation behaviour from language data and then compiles
|
||||
that behaviour into its runtime trie.
|
||||
|
||||
### It is not a full morphological analyzer
|
||||
|
||||
A full analyzer may return lemmas, parts of speech, grammatical tags, and
|
||||
multiple analyses. That is valuable when applications need morphological
|
||||
interpretation.
|
||||
|
||||
Radixor has a narrower search-oriented objective: produce compact, high-quality
|
||||
term conflation with predictable runtime cost. It can preserve multiple stemming
|
||||
candidates, but it does not attempt to become a general-purpose morphological
|
||||
analysis framework.
|
||||
|
||||
## What is distinctive about the combination
|
||||
|
||||
Any one ingredient in isolation is familiar:
|
||||
|
||||
- dictionaries are familiar;
|
||||
- tries are familiar;
|
||||
- string edit commands are familiar;
|
||||
- subtree reduction is familiar.
|
||||
|
||||
The distinctive architecture is their **combination**:
|
||||
|
||||
> lexical evidence → patch commands → trie organization → semantic reduction →
|
||||
> compact deterministic runtime transformation
|
||||
|
||||
That architecture separates expensive learning and compilation from hot-path
|
||||
runtime work.
|
||||
|
||||
## Modern Radixor adds more than the historical implementation
|
||||
|
||||
Radixor preserves the useful Egothor idea while rebuilding the operational model
|
||||
for current software:
|
||||
|
||||
- immutable compiled tries;
|
||||
- compiled patch-command objects rather than repeated textual interpretation;
|
||||
- deterministic ranked multi-result lookup;
|
||||
- configurable reduction semantics;
|
||||
- uniform-subtree contraction;
|
||||
- binary persistence;
|
||||
- independent language-model versioning and integrity verification;
|
||||
- reopening and extending compiled structures;
|
||||
- Java and Python native runtimes;
|
||||
- reproducible quality and performance benchmark infrastructure.
|
||||
|
||||
For the historical lineage and how Stempel, Morfologik, Snowball, and other
|
||||
comparators relate to this architecture, continue with
|
||||
[Technology and Lineage](technology-lineage.md).
|
||||
303
gradle/python.gradle
Normal file
303
gradle/python.gradle
Normal file
@@ -0,0 +1,303 @@
|
||||
def pythonProjectDirectory = layout.projectDirectory.dir('python')
|
||||
def pythonHostDistributionDirectory = layout.buildDirectory.dir('python/dist/host')
|
||||
def pythonSdistDistributionDirectory = layout.buildDirectory.dir('python/dist/sdist')
|
||||
def pythonStandardDistributionDirectory = layout.buildDirectory.dir('python/dist/standard')
|
||||
def pythonGeneratedStandardProjectDirectory = layout.buildDirectory.dir('python/generated/models-standard')
|
||||
def pythonBenchmarkRuntimeDirectory = layout.buildDirectory.dir('python/runtime/benchmark')
|
||||
def pythonModelCompilerRuntimeDirectory = layout.buildDirectory.dir('python/runtime/model-compiler')
|
||||
def pythonBenchmarkReportDirectory = layout.buildDirectory.dir('reports/python-benchmarks')
|
||||
def pythonTemporaryDirectory = layout.buildDirectory.dir('python/tmp')
|
||||
|
||||
def hostOsName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT)
|
||||
def hostPlatform = hostOsName.contains('win') ? 'windows'
|
||||
: hostOsName.contains('mac') || hostOsName.contains('darwin') ? 'macos'
|
||||
: 'linux'
|
||||
def hostArchitectureName = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT)
|
||||
def hostArchitecture = hostArchitectureName in ['aarch64', 'arm64'] ? 'aarch64' : 'x86_64'
|
||||
|
||||
def pythonExecutable = providers.gradleProperty('pythonExecutable')
|
||||
.orElse(hostPlatform == 'windows' ? 'python' : 'python3')
|
||||
def maturinExecutable = providers.gradleProperty('maturinExecutable').orElse('maturin')
|
||||
def pythonToolIdentity = providers.exec {
|
||||
commandLine(pythonExecutable.get(), '--version')
|
||||
}.standardOutput.asText.map { String value -> value.strip() }
|
||||
def maturinToolIdentity = providers.exec {
|
||||
commandLine(maturinExecutable.get(), '--version')
|
||||
}.standardOutput.asText.map { String value -> value.strip() }
|
||||
def rustToolIdentity = providers.exec {
|
||||
commandLine('rustc', '--version')
|
||||
}.standardOutput.asText.map { String value -> value.strip() }
|
||||
def pythonBenchmarkWords = providers.gradleProperty('pythonBenchmarkWords').orElse('5000')
|
||||
def pythonBenchmarkRepeats = providers.gradleProperty('pythonBenchmarkRepeats').orElse('15')
|
||||
def pythonBenchmarkWarmup = providers.gradleProperty('pythonBenchmarkWarmup').orElse('3')
|
||||
|
||||
def rustTargets = [
|
||||
linux : [x86_64: 'x86_64-unknown-linux-gnu', aarch64: 'aarch64-unknown-linux-gnu'],
|
||||
windows: [x86_64: 'x86_64-pc-windows-msvc', aarch64: 'aarch64-pc-windows-msvc'],
|
||||
macos : [x86_64: 'x86_64-apple-darwin', aarch64: 'aarch64-apple-darwin']
|
||||
]
|
||||
|
||||
def pythonBuildStandardModels = tasks.register('pythonBuildStandardModels', Exec) {
|
||||
group = 'python'
|
||||
description = 'Builds the pure py3-none-any standard-model wheel and offline-ready sdist.'
|
||||
|
||||
dependsOn('regeneratePythonStandardModels')
|
||||
inputs.dir(pythonGeneratedStandardProjectDirectory)
|
||||
inputs.file(pythonProjectDirectory.file('scripts/build_standard_distribution.py'))
|
||||
inputs.property('pythonExecutable', pythonExecutable)
|
||||
inputs.property('pythonToolIdentity', pythonToolIdentity)
|
||||
outputs.dir(pythonStandardDistributionDirectory)
|
||||
|
||||
workingDir(layout.projectDirectory)
|
||||
doFirst {
|
||||
final File output = pythonStandardDistributionDirectory.get().asFile
|
||||
if (output.exists() && !output.deleteDir()) {
|
||||
throw new GradleException("Cannot clean standard-model distribution directory: ${output}")
|
||||
}
|
||||
output.mkdirs()
|
||||
commandLine(pythonExecutable.get(), 'python/scripts/build_standard_distribution.py',
|
||||
'--project', pythonGeneratedStandardProjectDirectory.get().asFile.absolutePath,
|
||||
'--outdir', output.absolutePath)
|
||||
}
|
||||
}
|
||||
|
||||
def pythonBuildSdist = tasks.register('pythonBuildSdist', Exec) {
|
||||
group = 'python'
|
||||
description = 'Builds the Radixor native source distribution without runtime model data.'
|
||||
|
||||
inputs.files(pythonProjectDirectory.file('Cargo.toml'), pythonProjectDirectory.file('Cargo.lock'),
|
||||
pythonProjectDirectory.file('pyproject.toml'), layout.projectDirectory.file('gradle/python.gradle'))
|
||||
inputs.dir(pythonProjectDirectory.dir('src'))
|
||||
inputs.files(fileTree(pythonProjectDirectory.dir('radixor')) {
|
||||
exclude 'models/**'
|
||||
})
|
||||
inputs.property('maturinExecutable', maturinExecutable)
|
||||
inputs.property('maturinToolIdentity', maturinToolIdentity)
|
||||
outputs.dir(pythonSdistDistributionDirectory)
|
||||
|
||||
workingDir(pythonProjectDirectory)
|
||||
doFirst {
|
||||
final File output = pythonSdistDistributionDirectory.get().asFile
|
||||
if (output.exists() && !output.deleteDir()) {
|
||||
throw new GradleException("Cannot clean Python sdist output directory: ${output}")
|
||||
}
|
||||
output.mkdirs()
|
||||
commandLine(maturinExecutable.get(), 'sdist', '--out', output.absolutePath)
|
||||
}
|
||||
}
|
||||
|
||||
def registerPythonWheelBuild = { String taskName, String taskDescription, Provider<Directory> outputDirectory,
|
||||
String target ->
|
||||
tasks.register(taskName, Exec) {
|
||||
group = 'python'
|
||||
description = taskDescription
|
||||
|
||||
inputs.files(pythonProjectDirectory.file('Cargo.toml'), pythonProjectDirectory.file('Cargo.lock'),
|
||||
pythonProjectDirectory.file('pyproject.toml'), layout.projectDirectory.file('gradle/python.gradle'))
|
||||
inputs.dir(pythonProjectDirectory.dir('src'))
|
||||
inputs.dir(pythonProjectDirectory.dir('radixor'))
|
||||
inputs.property('maturinExecutable', maturinExecutable)
|
||||
inputs.property('maturinToolIdentity', maturinToolIdentity)
|
||||
inputs.property('rustToolIdentity', rustToolIdentity)
|
||||
inputs.property('pythonBuildTarget', target == null ? 'host' : target)
|
||||
if (target == null) {
|
||||
inputs.property('pythonExecutable', pythonExecutable)
|
||||
inputs.property('pythonToolIdentity', pythonToolIdentity)
|
||||
}
|
||||
outputs.dir(outputDirectory)
|
||||
|
||||
workingDir(pythonProjectDirectory)
|
||||
doFirst {
|
||||
final File output = outputDirectory.get().asFile
|
||||
if (output.exists() && !output.deleteDir()) {
|
||||
throw new GradleException("Cannot clean Python wheel output directory: ${output}")
|
||||
}
|
||||
output.mkdirs()
|
||||
final List<String> arguments = [
|
||||
'build', '--release', '--locked', '--out', output.absolutePath
|
||||
]
|
||||
if (target != null) {
|
||||
arguments.addAll(['--target', target])
|
||||
} else {
|
||||
arguments.addAll(['--interpreter', pythonExecutable.get()])
|
||||
}
|
||||
commandLine([maturinExecutable.get()] + arguments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def pythonBuildNativeWheel = registerPythonWheelBuild(
|
||||
'pythonBuildNativeWheel',
|
||||
'Builds the Radixor native Python wheel for the current host platform.',
|
||||
pythonHostDistributionDirectory,
|
||||
null
|
||||
)
|
||||
def pythonBuild = tasks.register('pythonBuild') {
|
||||
group = 'python'
|
||||
description = 'Builds the host native wheel/sdist and generated standard-model wheel/sdist.'
|
||||
dependsOn(pythonBuildNativeWheel, pythonBuildStandardModels, pythonBuildSdist)
|
||||
}
|
||||
|
||||
rustTargets.each { String platform, Map<String, String> architectureTargets ->
|
||||
final String taskName = 'pythonBuild' + platform.capitalize()
|
||||
if (platform == hostPlatform) {
|
||||
tasks.register(taskName) {
|
||||
group = 'python'
|
||||
description = "Builds the Radixor Python release wheel for ${platform} on the current host."
|
||||
dependsOn(pythonBuild)
|
||||
}
|
||||
} else {
|
||||
final String propertyName = 'python' + platform.capitalize() + 'Target'
|
||||
final String target = providers.gradleProperty(propertyName)
|
||||
.getOrElse(architectureTargets[hostArchitecture])
|
||||
registerPythonWheelBuild(
|
||||
taskName,
|
||||
"Cross-builds the Radixor Python release wheel for ${platform}; requires the target toolchain.",
|
||||
layout.buildDirectory.dir("python/dist/${platform}"),
|
||||
target
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('preparePythonBenchmarkRuntime', Sync) {
|
||||
group = 'python'
|
||||
description = 'Extracts the host Python wheel into an isolated benchmark runtime.'
|
||||
|
||||
dependsOn(pythonBuild)
|
||||
|
||||
from {
|
||||
final Set<File> wheels = fileTree(pythonHostDistributionDirectory).matching {
|
||||
include '*.whl'
|
||||
}.files
|
||||
if (wheels.size() != 1) {
|
||||
throw new GradleException("Expected exactly one host Python wheel, found ${wheels.size()} in "
|
||||
+ pythonHostDistributionDirectory.get().asFile)
|
||||
}
|
||||
zipTree(wheels.first())
|
||||
}
|
||||
from {
|
||||
final Set<File> wheels = fileTree(pythonStandardDistributionDirectory).matching {
|
||||
include '*.whl'
|
||||
}.files
|
||||
if (wheels.size() != 1) {
|
||||
throw new GradleException("Expected exactly one standard-model Python wheel, found ${wheels.size()} in "
|
||||
+ pythonStandardDistributionDirectory.get().asFile)
|
||||
}
|
||||
zipTree(wheels.first())
|
||||
}
|
||||
into(pythonBenchmarkRuntimeDirectory)
|
||||
}
|
||||
|
||||
tasks.register('preparePythonModelCompilerRuntime', Sync) {
|
||||
group = 'python'
|
||||
description = 'Extracts the host native wheel for deterministic standard-model regeneration.'
|
||||
dependsOn(pythonBuildNativeWheel)
|
||||
from {
|
||||
final Set<File> wheels = fileTree(pythonHostDistributionDirectory).matching {
|
||||
include '*.whl'
|
||||
}.files
|
||||
if (wheels.size() != 1) {
|
||||
throw new GradleException("Expected exactly one host Python wheel, found ${wheels.size()} in "
|
||||
+ pythonHostDistributionDirectory.get().asFile)
|
||||
}
|
||||
zipTree(wheels.first())
|
||||
}
|
||||
into(pythonModelCompilerRuntimeDirectory)
|
||||
}
|
||||
|
||||
tasks.register('regeneratePythonStandardModels', Exec) {
|
||||
group = 'python'
|
||||
description = 'Generates standard .rxc artifacts and metadata below build/ from canonical model sources.'
|
||||
dependsOn(tasks.named('preparePythonModelCompilerRuntime'))
|
||||
inputs.file(layout.projectDirectory.file('models/model-projects.properties'))
|
||||
inputs.file(layout.projectDirectory.file('models/catalog-version.txt'))
|
||||
inputs.files(fileTree(layout.projectDirectory.dir('models')) {
|
||||
include '*/build.gradle', '*/model-version.txt', '*/src/modelInput/stemmer.gz',
|
||||
'*/src/modelInput/NOTICE-model-data.txt'
|
||||
})
|
||||
inputs.files(fileTree(pythonProjectDirectory.dir('models-standard')) {
|
||||
exclude 'build/**', 'dist/**', '*.egg-info/**', '**/__pycache__/**',
|
||||
'radixor_models_standard/manifest.json', 'radixor_models_standard/models/*.rxc',
|
||||
'radixor_models_standard/notices/*/NOTICE-model-data.txt'
|
||||
})
|
||||
inputs.file(pythonProjectDirectory.file('scripts/build_standard_models.py'))
|
||||
outputs.dir(pythonGeneratedStandardProjectDirectory)
|
||||
workingDir(layout.projectDirectory)
|
||||
doFirst {
|
||||
environment('PYTHONPATH', pythonModelCompilerRuntimeDirectory.get().asFile.absolutePath)
|
||||
commandLine(
|
||||
pythonExecutable.get(), 'python/scripts/build_standard_models.py',
|
||||
'--project', pythonGeneratedStandardProjectDirectory.get().asFile.absolutePath,
|
||||
'--distribution-version', '0.0.0'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('pythonVerifyDistributions', Exec) {
|
||||
group = 'verification'
|
||||
description = 'Verifies both Python archives and a fresh offline wheel-only installation.'
|
||||
dependsOn(pythonBuild)
|
||||
inputs.dir(pythonHostDistributionDirectory)
|
||||
inputs.dir(pythonSdistDistributionDirectory)
|
||||
inputs.dir(pythonStandardDistributionDirectory)
|
||||
inputs.file(pythonProjectDirectory.file('scripts/verify_distributions.py'))
|
||||
outputs.upToDateWhen { false }
|
||||
workingDir(layout.projectDirectory)
|
||||
commandLine(
|
||||
pythonExecutable.get(), 'python/scripts/verify_distributions.py',
|
||||
'--main-wheel-dir', pythonHostDistributionDirectory.get().asFile.absolutePath,
|
||||
'--main-sdist-dir', pythonSdistDistributionDirectory.get().asFile.absolutePath,
|
||||
'--standard-dir', pythonStandardDistributionDirectory.get().asFile.absolutePath
|
||||
)
|
||||
}
|
||||
|
||||
tasks.register('pythonBenchmarkAllLanguagesBatch', Exec) {
|
||||
group = 'verification'
|
||||
description = 'Benchmarks the host Python wheel and available competitors for every supported language.'
|
||||
|
||||
dependsOn(tasks.named('preparePythonBenchmarkRuntime'))
|
||||
|
||||
inputs.files(pythonProjectDirectory.file('benchmarks/run_benchmark.py'),
|
||||
pythonProjectDirectory.file('benchmarks/corpus.py'),
|
||||
pythonProjectDirectory.file('benchmarks/engines.py'),
|
||||
layout.projectDirectory.file('gradle/python.gradle'))
|
||||
inputs.dir(pythonBenchmarkRuntimeDirectory)
|
||||
inputs.property('pythonExecutable', pythonExecutable)
|
||||
inputs.property('pythonToolIdentity', pythonToolIdentity)
|
||||
inputs.property('pythonBenchmarkWords', pythonBenchmarkWords)
|
||||
inputs.property('pythonBenchmarkRepeats', pythonBenchmarkRepeats)
|
||||
inputs.property('pythonBenchmarkWarmup', pythonBenchmarkWarmup)
|
||||
outputs.file(pythonBenchmarkReportDirectory.map { it.file('all-languages-batch.csv') })
|
||||
outputs.file(pythonBenchmarkReportDirectory.map { it.file('all-languages-batch.json') })
|
||||
outputs.upToDateWhen { false }
|
||||
|
||||
workingDir(pythonProjectDirectory)
|
||||
doFirst {
|
||||
final File reportDirectory = pythonBenchmarkReportDirectory.get().asFile
|
||||
reportDirectory.mkdirs()
|
||||
environment('PYTHONPATH', pythonBenchmarkRuntimeDirectory.get().asFile.absolutePath)
|
||||
commandLine(
|
||||
pythonExecutable.get(),
|
||||
'benchmarks/run_benchmark.py',
|
||||
'--all-languages',
|
||||
'--sizes', '10', '20', '50', '100',
|
||||
'--words', pythonBenchmarkWords.get(),
|
||||
'--repeats', pythonBenchmarkRepeats.get(),
|
||||
'--warmup', pythonBenchmarkWarmup.get(),
|
||||
'--csv', new File(reportDirectory, 'all-languages-batch.csv').absolutePath,
|
||||
'--json', new File(reportDirectory, 'all-languages-batch.json').absolutePath
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType(Exec).configureEach { Exec task ->
|
||||
if (task.name.startsWith('python') || task.name == 'regeneratePythonStandardModels') {
|
||||
task.doFirst {
|
||||
final File temporaryDirectory = pythonTemporaryDirectory.get().asFile
|
||||
temporaryDirectory.mkdirs()
|
||||
environment('TMPDIR', temporaryDirectory.absolutePath)
|
||||
environment('TEMP', temporaryDirectory.absolutePath)
|
||||
environment('TMP', temporaryDirectory.absolutePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
import org.gradle.plugins.ide.eclipse.model.SourceFolder
|
||||
|
||||
import java.security.MessageDigest
|
||||
|
||||
def snowballVersion = '3.0.1'
|
||||
|
||||
def snowballVersion = '3.1.0'
|
||||
def snowballArchiveSha256 = '5dab34d491f55f47b6e971569ffe6aadf5991512c648ddfe5d331b494cf6d655'
|
||||
def snowballArchiveName = "libstemmer_java-${snowballVersion}.tar.gz"
|
||||
def snowballDistributionDirectoryName = "libstemmer_java-${snowballVersion}"
|
||||
def snowballRootRelativePath = 'third-party/snowball'
|
||||
@@ -73,11 +76,32 @@ tasks.register('downloadSnowballJava') {
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('verifySnowballJava') {
|
||||
group = 'verification'
|
||||
description = 'Verifies the official Snowball Java source distribution checksum.'
|
||||
|
||||
dependsOn(tasks.named('downloadSnowballJava'))
|
||||
inputs.file(snowballDownloadFile)
|
||||
inputs.property('expectedSha256', snowballArchiveSha256)
|
||||
|
||||
doLast {
|
||||
final File archive = snowballDownloadFile.get().asFile
|
||||
final String actualSha256 = MessageDigest.getInstance('SHA-256')
|
||||
.digest(archive.bytes)
|
||||
.encodeHex()
|
||||
.toString()
|
||||
if (actualSha256 != snowballArchiveSha256) {
|
||||
throw new GradleException("Snowball ${snowballVersion} archive SHA-256 mismatch: "
|
||||
+ "expected ${snowballArchiveSha256}, found ${actualSha256}.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('extractSnowballJava', Copy) {
|
||||
group = 'build setup'
|
||||
description = 'Extracts the official Snowball Java source distribution.'
|
||||
|
||||
dependsOn(tasks.named('downloadSnowballJava'))
|
||||
dependsOn(tasks.named('verifySnowballJava'))
|
||||
|
||||
from(tarTree(resources.gzip(snowballDownloadFile)))
|
||||
into(snowballExtractDirectory)
|
||||
|
||||
50
mkdocs.yml
50
mkdocs.yml
@@ -1,15 +1,17 @@
|
||||
site_name: Radixor
|
||||
site_description: High-performance multi-language stemming toolkit for Java
|
||||
site_description: Learned transformation stemming with compiled patch-command tries for Java and Python
|
||||
site_url: https://leogalambos.github.io/Radixor/
|
||||
repo_url: https://github.com/leogalambos/Radixor
|
||||
repo_name: leogalambos/Radixor
|
||||
copyright: "© 2026 Egothor. Licensed under <a href='https://github.com/leogalambos/Radixor/blob/main/LICENSE'>BSD-3-Clause</a>."
|
||||
copyright: "© 2026 Egothor · Radixor software is BSD-3-Clause licensed."
|
||||
|
||||
theme:
|
||||
name: material
|
||||
language: en
|
||||
custom_dir: docs/overrides
|
||||
logo: assets/images/radixor-logo.png
|
||||
favicon: assets/images/radixor-logo.png
|
||||
features:
|
||||
- navigation.instant
|
||||
- navigation.sections
|
||||
- navigation.top
|
||||
- search.suggest
|
||||
@@ -17,14 +19,20 @@ theme:
|
||||
- content.code.copy
|
||||
palette:
|
||||
- scheme: default
|
||||
primary: indigo
|
||||
accent: indigo
|
||||
primary: white
|
||||
accent: blue
|
||||
|
||||
extra:
|
||||
extra:
|
||||
generator: false
|
||||
|
||||
extra_css:
|
||||
- assets/stylesheets/extra.css
|
||||
- assets/stylesheets/landing-v2.css
|
||||
- assets/stylesheets/radixor-docs-safety.css
|
||||
|
||||
extra_javascript:
|
||||
- https://unpkg.com/mermaid@11/dist/mermaid.min.js
|
||||
- assets/javascripts/mermaid.js
|
||||
|
||||
markdown_extensions:
|
||||
- admonition
|
||||
@@ -32,18 +40,28 @@ markdown_extensions:
|
||||
- md_in_html
|
||||
- pymdownx.details
|
||||
- pymdownx.highlight
|
||||
- pymdownx.superfences
|
||||
- pymdownx.superfences:
|
||||
custom_fences:
|
||||
- name: mermaid
|
||||
class: mermaid
|
||||
format: !!python/name:pymdownx.superfences.fence_code_format
|
||||
- pymdownx.tabbed:
|
||||
alternate_style: true
|
||||
- tables
|
||||
|
||||
nav:
|
||||
- Home: index.md
|
||||
|
||||
- Start:
|
||||
- Fast Track: fast-track.md
|
||||
- Quick Start: quick-start.md
|
||||
- Integration Deep Dive: integration-deep-dive.md
|
||||
- Choose a Runtime: getting-started.md
|
||||
- Why Radixor Is Different: why-radixor-is-different.md
|
||||
- Python Fast Track: python/fast-track.md
|
||||
- Java Fast Track: fast-track.md
|
||||
- Python Quick Start: python/quick-start.md
|
||||
- Java Quick Start: quick-start.md
|
||||
|
||||
- Integration:
|
||||
- Java:
|
||||
- Integration Deep Dive: integration-deep-dive.md
|
||||
- Overview: programmatic-usage.md
|
||||
- Model Selection and Loading: model-selection-and-loading.md
|
||||
- Loading and Building Stemmers: programmatic-loading-and-building.md
|
||||
@@ -52,6 +70,15 @@ nav:
|
||||
- Migration and Backward Compatibility: migration-and-backward-compatibility.md
|
||||
- CLI Compilation: cli-compilation.md
|
||||
|
||||
- Python:
|
||||
- Fast Track: python/fast-track.md
|
||||
- Quick Start: python/quick-start.md
|
||||
- Overview: python/index.md
|
||||
- Installation and Builds: python/installation.md
|
||||
- Usage and API: python/usage.md
|
||||
- Dictionary Compilation: python/model-compilation.md
|
||||
- Benchmarks: python/performance.md
|
||||
|
||||
- Dictionaries and Languages:
|
||||
- Stemmer Models: stemmer-models.md
|
||||
- Published Model Catalog: stemmer-model-catalog.md
|
||||
@@ -60,6 +87,7 @@ nav:
|
||||
- Contributing Dictionaries: contributing-dictionaries.md
|
||||
|
||||
- Architecture and Semantics:
|
||||
- Technology and Lineage: technology-lineage.md
|
||||
- Overview: architecture-and-reduction.md
|
||||
- Architecture: architecture.md
|
||||
- Reduction Semantics: reduction-semantics.md
|
||||
|
||||
23
python/.gitignore
vendored
Normal file
23
python/.gitignore
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
# Rust and maturin build products
|
||||
/target/
|
||||
**/dist/
|
||||
/models-standard/build/
|
||||
/radixor/models/
|
||||
*.so
|
||||
*.egg-info/
|
||||
|
||||
# Standard-model payload is generated deterministically below the repository
|
||||
# build directory and belongs only in the published wheel/sdist.
|
||||
/models-standard/radixor_models_standard/manifest.json
|
||||
/models-standard/radixor_models_standard/models/*.rxc
|
||||
/models-standard/radixor_models_standard/notices/*/NOTICE-model-data.txt
|
||||
|
||||
# Local Python environments and caches
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
|
||||
# Reports produced by direct benchmark-script runs
|
||||
/benchmarks/results*.csv
|
||||
/benchmarks/results*.json
|
||||
254
python/Cargo.lock
generated
Normal file
254
python/Cargo.lock
generated
Normal file
@@ -0,0 +1,254 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "indoc"
|
||||
version = "2.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3"
|
||||
version = "0.22.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"indoc",
|
||||
"libc",
|
||||
"memoffset",
|
||||
"once_cell",
|
||||
"portable-atomic",
|
||||
"pyo3-build-config",
|
||||
"pyo3-ffi",
|
||||
"pyo3-macros",
|
||||
"unindent",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-build-config"
|
||||
version = "0.22.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-ffi"
|
||||
version = "0.22.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pyo3-build-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros"
|
||||
version = "0.22.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"pyo3-macros-backend",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros-backend"
|
||||
version = "0.22.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"pyo3-build-config",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "radixor"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"pyo3",
|
||||
"unicode-general-category",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-general-category"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unindent"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
||||
20
python/Cargo.toml
Normal file
20
python/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "radixor"
|
||||
version = "0.0.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
name = "_radixor"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
pyo3 = { version = "0.22", features = ["extension-module", "abi3-py39"] }
|
||||
flate2 = "1.0"
|
||||
unicode-normalization = "0.1"
|
||||
unicode-general-category = "0.6"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
strip = true
|
||||
28
python/LICENSE
Normal file
28
python/LICENSE
Normal file
@@ -0,0 +1,28 @@
|
||||
Copyright (C) 2026, Leo Galambos
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
253
python/README.md
Normal file
253
python/README.md
Normal file
@@ -0,0 +1,253 @@
|
||||
# radixor — Fastest Stemming for Python
|
||||
|
||||
**radixor** is a Python extension for the [Radixor](https://github.com/leogalambos/Radixor) stemmer library, built on a Rust core via [PyO3](https://pyo3.rs/). It provides sub-microsecond per-word stemming with a batch API that amortises the Python↔Rust bridge overhead across thousands of words at once.
|
||||
|
||||
## Why radixor?
|
||||
|
||||
| Library | Approach | Batch API |
|
||||
|---|---|---|
|
||||
| **radixor** | Compiled patch-command trie in Rust | ✅ `stem_batch()` |
|
||||
| PyStemmer (Snowball) | C extension (`libstemmer`) | ✅ `stemWords()` |
|
||||
| snowballstemmer | Pure-Python Snowball | ✅ (Python loop) |
|
||||
| NLTK Porter / CISTEM | Pure Python | ❌ |
|
||||
|
||||
**Performance.** On the shared UniMorph gold-standard corpus, measuring runtime
|
||||
stemming only (construction excluded) and with a fair, cache-disabled,
|
||||
same-input methodology, radixor won all **18 / 18** direct comparisons with
|
||||
PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run.
|
||||
At batch size 100, the geometric-mean speedup was **1.67×**. The complete
|
||||
machine metadata and current results are in the [Python performance
|
||||
documentation](../docs/python/performance.md); benchmark implementation and
|
||||
fairness notes are in [`benchmarks/`](benchmarks/README.md).
|
||||
|
||||
## Installation
|
||||
|
||||
From PyPI, once publication is enabled:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: radixor
|
||||
```
|
||||
|
||||
The GitHub Releases-backed index is the independent alternative:
|
||||
|
||||
```bash
|
||||
python -m pip install --only-binary=:all: \
|
||||
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
|
||||
```
|
||||
|
||||
The GitHub command becomes usable after the first model and native releases
|
||||
populate that index. See the [installation guide](../docs/python/installation.md)
|
||||
for current availability and source-checkout builds.
|
||||
|
||||
Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install
|
||||
also resolves the mandatory pure `radixor-models-standard` dependency
|
||||
with 20 precompiled standard models. Building the native source distribution
|
||||
requires Rust ≥ 1.75 and [maturin](https://www.maturin.rs/).
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from radixor import Stemmer
|
||||
|
||||
s = Stemmer("en") # English (us-uk-default model)
|
||||
s.stem("running") # → "run"
|
||||
s.stem("cats") # → "cat"
|
||||
s.stem("unknown_word") # → None
|
||||
```
|
||||
|
||||
## Batch API — the fast path
|
||||
|
||||
```python
|
||||
words = ["running", "cats", "stemming", "quickly"]
|
||||
|
||||
# Amortises the Python→Rust bridge cost across all words at once
|
||||
stems = s.stem_batch(words)
|
||||
# → ["run", "cat", "stem", "quick"]
|
||||
```
|
||||
|
||||
For large corpora (tens of thousands of words) the batch call is the recommended interface. It avoids per-call Python frame overhead and keeps the hot loop entirely inside Rust.
|
||||
|
||||
## Migrating from PyStemmer
|
||||
|
||||
Radixor provides PyStemmer's `stemWord` and `stemWords` method names. These
|
||||
compatibility methods also follow PyStemmer's fallback behavior: when the trie
|
||||
has no patch command, they return the original word instead of `None`.
|
||||
|
||||
```python
|
||||
# PyStemmer: import Stemmer
|
||||
import radixor as Stemmer
|
||||
|
||||
stemmer = Stemmer.Stemmer("english")
|
||||
stemmer.stemWord("running") # → "run"
|
||||
stemmer.stemWord("unknown_word") # → "unknown_word"
|
||||
stemmer.stemWords(["running", "unknown"]) # → ["run", "unknown"]
|
||||
```
|
||||
|
||||
The original Radixor methods remain unchanged: `stem` and `stem_batch` return
|
||||
`None` for words without a matching patch command. Radixor accepts PyStemmer's
|
||||
full language names for the languages represented by its bundled models, as
|
||||
well as its existing two-letter codes and model IDs.
|
||||
|
||||
## Supported languages
|
||||
|
||||
| Code | Language | Model ID |
|
||||
|---|---|---|
|
||||
| `cs` | Czech | `cs-cz-default` |
|
||||
| `da` | Danish | `da-dk-default` |
|
||||
| `de` | German | `de-de-default` |
|
||||
| `en` | English | `us-uk-default` |
|
||||
| `es` | Spanish | `es-es-default` |
|
||||
| `fa` | Persian | `fa-ir-default` |
|
||||
| `fi` | Finnish | `fi-fi-default` |
|
||||
| `fr` | French | `fr-fr-default` |
|
||||
| `he` | Hebrew | `he-il-default` |
|
||||
| `hu` | Hungarian | `hu-hu-default` |
|
||||
| `it` | Italian | `it-it-default` |
|
||||
| `nb` | Norwegian Bokmål | `nb-no-default` |
|
||||
| `nl` | Dutch | `nl-nl-default` |
|
||||
| `nn` | Norwegian Nynorsk | `nn-no-default` |
|
||||
| `pl` | Polish | `pl-pl-unimorph` |
|
||||
| `pt` | Portuguese | `pt-pt-default` |
|
||||
| `ru` | Russian | `ru-ru-default` |
|
||||
| `sv` | Swedish | `sv-se-default` |
|
||||
| `uk` | Ukrainian | `uk-ua-default` |
|
||||
| `yi` | Yiddish | `yi-default` |
|
||||
|
||||
## API reference
|
||||
|
||||
### `Stemmer(language=None, *, path=None, compiled=None, backward=None, store_original=True, lowercase=True, cache_size=10_000)`
|
||||
|
||||
Create a stemmer for the given language code, model ID, custom textual
|
||||
dictionary, or previously compiled version 7 trie. Textual dictionaries are
|
||||
compiled in Rust; `compiled=` loads a prepared binary directly.
|
||||
|
||||
```python
|
||||
s = Stemmer("de") # by language code
|
||||
s = Stemmer("de-de-default") # by model ID
|
||||
s = Stemmer(path="/data/custom.gz") # custom gzipped dictionary
|
||||
s = Stemmer(compiled="/data/custom.rxc") # prepared v7 binary
|
||||
```
|
||||
|
||||
`backward` selects the traversal direction; when left as `None` it is derived
|
||||
from the language (BACKWARD, except right-to-left `fa`/`he`/`yi` which use
|
||||
FORWARD). `store_original` (default `True`) maps each canonical stem to a no-op
|
||||
patch so the stem itself is recognised. `lowercase=False` skips runtime
|
||||
lowercasing for already-normalized input, and `cache_size` enables the bounded
|
||||
result cache. The default holds up to 10,000 entries, matching PyStemmer;
|
||||
`cache_size=0` disables it. One cache is shared by `stem()`, `stemWord()`,
|
||||
`stem_batch()`, and `stemWords()`; the `stem_all*()` methods are not cached.
|
||||
|
||||
### `stem(word: str) → str | None`
|
||||
|
||||
Return the stem, or `None` when the compiled trie finds no applicable patch
|
||||
command. This does not mean that lookup is restricted to exact training words.
|
||||
|
||||
### `stem_batch(words: list[str]) → list[str | None]`
|
||||
|
||||
Stem an entire list. Preferred for large inputs.
|
||||
|
||||
### `stemWord(word: str) → str`
|
||||
|
||||
PyStemmer-compatible scalar method. Return the original word if it cannot be
|
||||
stemmed.
|
||||
|
||||
### `stemWords(words: list[str]) → list[str]`
|
||||
|
||||
PyStemmer-compatible batch method. Return each unrecognized word unchanged.
|
||||
|
||||
### `stem_all(word: str) → list[str]`
|
||||
|
||||
Return all stems ordered by descending corpus frequency. Useful when multiple valid stems exist.
|
||||
|
||||
### `stem_all_batch(words: list[str]) → list[list[str]]`
|
||||
|
||||
Return all stems for each word in a batch.
|
||||
|
||||
## Compiling a model (compile once, load instantly)
|
||||
|
||||
Compiling the trie from a textual dictionary takes time for large languages
|
||||
(seconds). You can compile it **once** to Radixor's binary format and then load
|
||||
it near-instantly — the same workflow Java users have:
|
||||
|
||||
```python
|
||||
import radixor
|
||||
radixor.compile("stemmer.gz", "en.rxc", language="en") # or backward=True/False
|
||||
s = radixor.Stemmer(compiled="en.rxc") # instant load, no re-compile
|
||||
```
|
||||
|
||||
The compiled file uses Radixor's **v7 trie format and is byte-compatible with
|
||||
the Java `StemmerPatchTrieBinaryIO`** (the inner stream is identical), so a file
|
||||
compiled by Java can be loaded by Python and vice versa. `Stemmer(path=...)`
|
||||
auto-detects whether it was given a compiled trie or a textual dictionary.
|
||||
|
||||
## Using a custom model
|
||||
|
||||
Provide your own gzipped source dictionary (tab-separated
|
||||
`stem<TAB>variant1<TAB>variant2…` per line, `#` / `//` line remarks allowed) and
|
||||
load it directly:
|
||||
|
||||
```python
|
||||
s = Stemmer(path="my_dictionary.gz") # BACKWARD by default
|
||||
s = Stemmer(path="my_rtl_dictionary.gz", backward=False) # right-to-left
|
||||
```
|
||||
|
||||
The dictionary is compiled to a patch-command trie in Rust at construction time.
|
||||
|
||||
## Building from source
|
||||
|
||||
```bash
|
||||
cd Radixor/
|
||||
pip install maturin build setuptools wheel pytest
|
||||
./gradlew pythonBuildStandardModels
|
||||
pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
|
||||
cd python/
|
||||
maturin develop --release # editable install with release optimisations
|
||||
```
|
||||
|
||||
From the repository root, Gradle builds the native wheel/sdist and pure
|
||||
standard-model wheel/sdist without installing them globally:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBuild
|
||||
```
|
||||
|
||||
The convenience tasks `pythonBuildLinux`, `pythonBuildWindows`, and
|
||||
`pythonBuildMacos` use the host build when the requested platform matches the
|
||||
current system. Other platforms are cross-compiled with the corresponding Rust
|
||||
target and therefore require that target and its linker/SDK to be installed.
|
||||
Override a default target with, for example,
|
||||
`-PpythonWindowsTarget=x86_64-pc-windows-gnu`. Build artifacts are written below
|
||||
`build/python/dist/`.
|
||||
|
||||
The complete batch benchmark runs Radixor for all bundled languages and every
|
||||
available comparison engine for the languages it supports, using batch sizes
|
||||
10, 20, 50, and 100:
|
||||
|
||||
Comparison engines are auto-detected in the environment of `pythonExecutable`.
|
||||
Install `python/benchmarks/requirements-bench.txt` there to enable the complete
|
||||
comparison set.
|
||||
|
||||
```bash
|
||||
./gradlew pythonBenchmarkAllLanguagesBatch
|
||||
```
|
||||
|
||||
Use `pythonBenchmarkWords`, `pythonBenchmarkRepeats`, and
|
||||
`pythonBenchmarkWarmup` Gradle properties to tune the run. CSV and JSON reports
|
||||
are written below `build/reports/python-benchmarks/`.
|
||||
|
||||
Neither runtime distribution contains textual dictionaries. The standard data
|
||||
sdist contains build-ready gzip v7 `.rxc` files, a checksummed provenance
|
||||
manifest, and per-model CC BY-SA 3.0 notices. They are generated below `build/`
|
||||
from canonical `models/*/src/modelInput/stemmer.gz` inputs and are never stored
|
||||
in Git. `./gradlew regeneratePythonStandardModels` performs this deterministic
|
||||
generation; repository topology selects the 20 defaults and excludes optional
|
||||
`pl-pl-polimorf`.
|
||||
|
||||
`radixor` requires `radixor-models-standard>=1.0,<2.0`. The Python distribution
|
||||
version is independent of its `2026.1` Java model-catalog identity and of the
|
||||
individual model versions recorded in the manifest.
|
||||
|
||||
## License
|
||||
|
||||
The native/API package is BSD-3-Clause — see [LICENSE](LICENSE). Model data
|
||||
is separately licensed under CC BY-SA 3.0 in its packaged notices.
|
||||
111
python/benchmarks/README.md
Normal file
111
python/benchmarks/README.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Runtime stemming benchmarks
|
||||
|
||||
These scripts measure **runtime stemming throughput only** — model construction
|
||||
and dictionary compilation happen once during setup and are **excluded** from
|
||||
every timing. Anyone can reproduce the numbers on their own machine.
|
||||
|
||||
## What is measured, and why batch sizes
|
||||
|
||||
Each engine is driven through its batch entry point over a fixed word budget
|
||||
(default 5 000 tokens), split into batches of **10, 20, 50, 100** words. The
|
||||
harness fits the descriptive line `per_call(N) ≈ intercept + N · slope` across
|
||||
those sizes. The fit is unconstrained and timing noise can make its intercept
|
||||
negative, so it describes observed scaling rather than physically separating
|
||||
overhead from word work. We report the *best* (minimum) of many repeats — the
|
||||
microbenchmark convention that suppresses OS/GC noise.
|
||||
|
||||
## Data — identical to the Java JMH benchmarks
|
||||
|
||||
The workload is the **changed-token corpus** built from the repository's
|
||||
canonical dictionaries (`models/<model>/src/modelInput/stemmer.gz`),
|
||||
mirroring `LanguageBenchmarkCorpus` in the Java project: each dictionary field
|
||||
is paired with its line's root, normalized `trim().lower()`, and only tokens
|
||||
that **differ** from their root are kept, in dictionary order, padded to ≥ 5 000
|
||||
tokens. See `corpus.py`.
|
||||
|
||||
## Fairness — three things that quietly break stemmer comparisons
|
||||
|
||||
Getting a *fair* comparison turned out to matter more than any micro-optimization.
|
||||
Three asymmetries, if left in, make the numbers meaningless:
|
||||
|
||||
1. **Result caching.** PyStemmer caches results by default (`maxCacheSize=10000`).
|
||||
Because a benchmark stems the same corpus every repeat, that cache turns
|
||||
measured passes into dict lookups rather than stemming. **The harness
|
||||
explicitly disables both caches**: PyStemmer uses `maxCacheSize=0` and
|
||||
radixor uses `cache_size=0`, so both engines do real stemming.
|
||||
snowballstemmer-pure, nltk-porter, and cistem have no cache.
|
||||
|
||||
2. **Lowercasing.** Snowball/PyStemmer do no case handling — they assume the
|
||||
caller pre-lowercased the input (our corpus is pre-lowercased for everyone).
|
||||
radixor normally lowercases internally; for a same-work comparison the
|
||||
harness runs radixor with **`lowercase=False`** (assume-already-lowercased),
|
||||
so Snowball and radixor do identical normalization work on identical input.
|
||||
CISTEM always performs its own lowercasing and German umlaut normalization;
|
||||
that unavoidable extra work modestly biases its comparison in radixor's
|
||||
favour.
|
||||
|
||||
3. **Delegation.** `snowballstemmer` delegates to PyStemmer when it is installed
|
||||
(they become the same C code). The harness bypasses that and uses
|
||||
snowballstemmer's genuine pure-Python backend, and records each engine's
|
||||
backing module + whether it is a compiled extension (`--json`) as proof.
|
||||
|
||||
## Engines compared
|
||||
|
||||
| Engine | Implementation | Batch API |
|
||||
|---|---|---|
|
||||
| `radixor` | Rust patch-command trie (cache disabled) | `stem_batch` — one FFI call per batch |
|
||||
| `PyStemmer` | Snowball C `libstemmer` (cache disabled) | `stemWords(list)` — one C call per batch |
|
||||
| `snowballstemmer-pure` | Official **pure-Python** Snowball | `stemWords(list)` — Python loop |
|
||||
| `nltk-porter` | Porter (English), pure Python | scalar loop |
|
||||
| `cistem` | CISTEM (German), pure Python (`nltk`) | scalar loop |
|
||||
|
||||
## Reproduce
|
||||
|
||||
```bash
|
||||
cd python/
|
||||
maturin develop --release # build the radixor extension into your env
|
||||
cd ..
|
||||
./gradlew pythonBuildStandardModels # generate and package standard models
|
||||
pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl
|
||||
cd python/
|
||||
pip install -r benchmarks/requirements-bench.txt
|
||||
python benchmarks/run_benchmark.py --language en de fr ru fi \
|
||||
--sizes 10 20 50 100 --repeats 21 \
|
||||
--json benchmarks/results.json --csv benchmarks/results.csv
|
||||
```
|
||||
|
||||
From the repository root, the Gradle integration builds an isolated host wheel
|
||||
and benchmarks Radixor plus every available comparison engine over all
|
||||
supported languages with the fixed 10/20/50/100 size sweep:
|
||||
|
||||
```bash
|
||||
./gradlew pythonBenchmarkAllLanguagesBatch
|
||||
```
|
||||
|
||||
The generated CSV and JSON reports are placed in
|
||||
`build/reports/python-benchmarks/`.
|
||||
|
||||
Comparison engines are auto-detected in the environment selected by the Gradle
|
||||
`pythonExecutable` property. Install `requirements-bench.txt` in that environment
|
||||
to enable the complete comparison set.
|
||||
|
||||
The run prints machine/Python/engine versions and each engine's backing module,
|
||||
and writes per-point rows (CSV) plus the full report incl. environment and
|
||||
provenance (JSON), so results are self-describing and verifiable.
|
||||
|
||||
## Published results
|
||||
|
||||
The canonical, current single-machine results and complete environment metadata
|
||||
are published on the documentation site's [Python performance
|
||||
page](../../docs/python/performance.md). Keeping the measured table in one place
|
||||
prevents results from different CPUs or benchmark runs from being mixed.
|
||||
|
||||
## Interpretation
|
||||
|
||||
- In the published 2026-08-08 run, **radixor is the fastest stemmer measured in
|
||||
Python** in all 18 languages directly shared with PyStemmer.
|
||||
- The benchmark intentionally disables caches. Cached-operation performance is
|
||||
outside this suite and must not be inferred from its results.
|
||||
- radixor and Snowball remain different *classes* of stemmer: radixor is
|
||||
dictionary-based (UniMorph gold coverage), Snowball is rule-based. radixor
|
||||
gives dictionary-quality stems *and* the best measured throughput.
|
||||
103
python/benchmarks/corpus.py
Normal file
103
python/benchmarks/corpus.py
Normal file
@@ -0,0 +1,103 @@
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Deterministic benchmark corpus, mirroring the Java JMH LanguageBenchmarkCorpus.
|
||||
|
||||
The timing workload contains dictionary forms that differ from their canonical
|
||||
root. Each field is normalized with ``strip().lower()``, retained when it needs
|
||||
stemming, and repeated in stable dictionary order until the corpus reaches
|
||||
``MINIMUM_TIMING_TOKEN_COUNT``. This mirrors
|
||||
``LanguageBenchmarkCorpus.buildChangedTimingCorpus`` and supplies the same
|
||||
token sequence to every benchmark engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
from pathlib import Path
|
||||
|
||||
MINIMUM_TIMING_TOKEN_COUNT = 5_000
|
||||
|
||||
|
||||
def _normalize(token: str) -> str:
|
||||
# Java: token.trim().toLowerCase(Locale.ROOT)
|
||||
return token.strip().lower()
|
||||
|
||||
|
||||
def _contains_whitespace(token: str) -> bool:
|
||||
return any(ch.isspace() for ch in token)
|
||||
|
||||
|
||||
def read_changed_tokens(dict_gz_path: str | Path) -> list[str]:
|
||||
"""Return the changed-token list (token != root) in dictionary order.
|
||||
|
||||
Not yet padded to the timing minimum; see :func:`build_timing_corpus`.
|
||||
"""
|
||||
tokens: list[str] = []
|
||||
with gzip.open(dict_gz_path, "rt", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
if not line or line.isspace():
|
||||
continue
|
||||
# Match the Java benchmark: only a leading marker starts a comment;
|
||||
# inline markers remain part of the dictionary field.
|
||||
if line.startswith("#") or line.startswith("//"):
|
||||
continue
|
||||
fields = line.split("\t")
|
||||
if not fields:
|
||||
continue
|
||||
root = _normalize(fields[0])
|
||||
if not root or _contains_whitespace(root):
|
||||
continue
|
||||
for field in fields:
|
||||
token = _normalize(field)
|
||||
if not token or _contains_whitespace(token):
|
||||
continue
|
||||
if token != root: # changed-token workload
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
|
||||
def build_timing_corpus(
|
||||
dict_gz_path: str | Path,
|
||||
minimum_token_count: int = MINIMUM_TIMING_TOKEN_COUNT,
|
||||
) -> list[str]:
|
||||
"""Return the padded changed-token timing corpus (>= minimum_token_count)."""
|
||||
changed = read_changed_tokens(dict_gz_path)
|
||||
if not changed:
|
||||
raise ValueError(f"No changed-token corpus tokens available in {dict_gz_path}")
|
||||
if len(changed) >= minimum_token_count:
|
||||
return changed
|
||||
# Repeat in stable order to reach the minimum, exactly like the Java code.
|
||||
out: list[str] = []
|
||||
n = len(changed)
|
||||
for i in range(minimum_token_count):
|
||||
out.append(changed[i % n])
|
||||
return out
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user