diff --git a/.github/RELEASING-PYTHON.md b/.github/RELEASING-PYTHON.md new file mode 100644 index 0000000..5853038 --- /dev/null +++ b/.github/RELEASING-PYTHON.md @@ -0,0 +1,52 @@ +# Releasing the Python distributions + +This maintainer-only checklist is intentionally outside the public MkDocs site. + +The repository descriptors use `0.0.0` as a non-release placeholder. Release +workflows create isolated staging trees and inject the stable version selected +by the tag. Never commit a release-number rewrite of the descriptors. +Standard `.rxc` resources are also generated in that staging tree from the +canonical model sources. The workflow compiles every model twice and rejects +non-deterministic output; generated payload must never be added to Git. + +## Validate without publishing + +Run **Python Standard Models Release** manually with version `1.0.0`, then run +**Python Native Release** with version `4.1.0`. `workflow_dispatch` validates +artifacts but cannot publish. The native run must pass Linux x86-64, Linux +ARM64, macOS universal2, and Windows x86-64. + +For the Linux paths, maintainers can use `act` with rootless Podman and the +event files under `.github/act/`. Do not pass production secrets to `act`. + +## Publish in dependency order + +Both tags must point to a commit already contained in `main`. + +```bash +git tag -a 'python-models-standard@1.0.0' \ + -m 'Python standard models 1.0.0' +git push origin 'python-models-standard@1.0.0' +``` + +Wait until the models workflow has published its GitHub Release and Pages +index entry. Then publish the native distribution: + +```bash +git tag -a 'python@4.1.0' -m 'Python Radixor 4.1.0' +git push origin 'python@4.1.0' +``` + +Do not push both tags together: native publication requires the standard-model +release to exist first. + +## Artifact identity + +Python releases use `SHA256SUMS` and GitHub keyless build-provenance +attestations. They do not use the Java Maven OpenPGP key. Verify a downloaded +artifact with: + +```bash +sha256sum --check SHA256SUMS +gh attestation verify --repo leogalambos/Radixor +``` diff --git a/.github/act/python-models-standard.json b/.github/act/python-models-standard.json new file mode 100644 index 0000000..7dfd71e --- /dev/null +++ b/.github/act/python-models-standard.json @@ -0,0 +1,5 @@ +{ + "inputs": { + "version": "1.0.0" + } +} diff --git a/.github/act/python-native.json b/.github/act/python-native.json new file mode 100644 index 0000000..0dfd6f0 --- /dev/null +++ b/.github/act/python-native.json @@ -0,0 +1,5 @@ +{ + "inputs": { + "version": "4.1.0" + } +} diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 51bd56e..9c39517 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -26,42 +26,43 @@ on: workflow_dispatch: permissions: - contents: write + contents: read concurrency: - group: pages-${{ github.ref }} - cancel-in-progress: true + group: github-python-pages + cancel-in-progress: false jobs: - publish-pages: - name: Publish static reports + build-pages: + name: Build static reports runs-on: ubuntu-latest steps: - name: Check out source repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 + persist-credentials: false - name: Validate Gradle wrapper - uses: gradle/actions/wrapper-validation@v4 + uses: gradle/actions/wrapper-validation@0b6dd653ba04f4f93bf581ec31e66cbd7dcb644d # v4 - name: Set up Temurin JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 with: distribution: temurin java-version: '21' - name: Set up Gradle caching and instrumentation - uses: gradle/actions/setup-gradle@v4 + uses: gradle/actions/setup-gradle@0b6dd653ba04f4f93bf581ec31e66cbd7dcb644d # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.x' + python-version: '3.14.6' - name: Install MkDocs Material - run: python -m pip install --upgrade pip mkdocs-material + run: python -m pip install --disable-pip-version-check mkdocs-material==9.7.6 - name: Verify reproducibility inputs shell: bash @@ -326,18 +327,80 @@ jobs: run: | set -euo pipefail mkdocs build --strict --config-file build/mkdocs/mkdocs.yml - rsync -a --delete --exclude '.git' --exclude '.git/' --exclude 'builds/' build/mkdocs-site/ .gh-pages/ + rsync -a --delete --exclude '.git' --exclude '.git/' --exclude 'builds/' --exclude 'python/' build/mkdocs-site/ .gh-pages/ mkdir -p .gh-pages/builds cp build/mkdocs-site/builds/index.html .gh-pages/builds/index.html cat > .gh-pages/.nojekyll <&2 + exit 1 + fi + - name: Upload static-site candidate + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: pages-site-${{ github.run_id }} + path: .gh-pages + if-no-files-found: error + include-hidden-files: true + retention-days: 1 + + publish-pages: + name: Publish static reports + needs: build-pages + runs-on: ubuntu-latest + environment: python-github-pages + permissions: + contents: write + + steps: + - name: Check out repository for publication + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + path: pages + + - name: Select or initialize gh-pages + shell: bash + run: | + set -euo pipefail + cd pages + if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then + git fetch origin gh-pages:refs/remotes/origin/gh-pages + git checkout -B gh-pages origin/gh-pages + else + git checkout --orphan gh-pages + git rm -rf . + fi + + - name: Download static-site candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: pages-site-${{ github.run_id }} + path: candidate + + - name: Validate and publish static site + shell: bash + run: | + set -euo pipefail + test ! -e candidate/python + test ! -e candidate/.git + if find candidate -type l -print -quit | grep -q .; then + echo 'Publication candidate contains a symbolic link.' >&2 + exit 1 + fi + rsync -a --delete \ + --exclude '.git' --exclude '.git/' \ + --exclude 'python' --exclude 'python/' candidate/ pages/ + cd pages git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" @@ -348,4 +411,4 @@ jobs: fi git commit -m "Publish reports for run ${GITHUB_RUN_NUMBER}" - git push origin gh-pages + git push origin HEAD:gh-pages diff --git a/.github/workflows/python-models-standard-release.yml b/.github/workflows/python-models-standard-release.yml new file mode 100644 index 0000000..4912d86 --- /dev/null +++ b/.github/workflows/python-models-standard-release.yml @@ -0,0 +1,235 @@ +name: Python Standard Models Release + +on: + push: + tags: + - 'python-models-standard@*' + workflow_dispatch: + inputs: + version: + description: Stable distribution version to validate without publishing + required: true + default: '1.0.0' + type: string + +permissions: + contents: read + +concurrency: + group: github-python-pages + cancel-in-progress: false + +jobs: + build: + name: Build and verify standard models + runs-on: ubuntu-latest + outputs: + version: ${{ steps.release.outputs.version }} + tag: ${{ steps.release.outputs.tag }} + + steps: + - name: Check out repository + if: ${{ env.ACT != 'true' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12.13' + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.88.0 + + - name: Select and validate release + id: release + shell: bash + env: + REQUESTED_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then + tag="${GITHUB_REF_NAME}" + else + tag="python-models-standard@${REQUESTED_VERSION}" + fi + eval "$(./tools/parse-python-release-tag.sh "${tag}")" + [[ "${PYTHON_DISTRIBUTION}" == 'radixor-models-standard' ]] + if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then + [[ "$(git rev-parse "${tag}^{commit}")" == "${GITHUB_SHA}" ]] + git merge-base --is-ancestor "${GITHUB_SHA}" origin/main + fi + printf 'version=%s\ntag=%s\n' "${PYTHON_VERSION}" "${tag}" >> "${GITHUB_OUTPUT}" + + - name: Install pinned build tools + run: >- + python -m pip install --disable-pip-version-check + maturin==1.14.1 setuptools==80.9.0 wheel==0.45.1 + + - name: Compile models and build isolated release tree + shell: bash + run: | + set -euo pipefail + rm -rf build/python-release + mkdir -p build/python-release/compiler-wheel build/python-release/compiler-runtime + maturin build --release --locked --manifest-path python/Cargo.toml \ + --out build/python-release/compiler-wheel + python -c "from pathlib import Path; import zipfile; wheels=list(Path('build/python-release/compiler-wheel').glob('*.whl')); assert len(wheels) == 1; zipfile.ZipFile(wheels[0]).extractall('build/python-release/compiler-runtime')" + PYTHONPATH=build/python-release/compiler-runtime \ + python python/scripts/build_standard_models.py \ + --project build/python-release/models-standard \ + --distribution-version '${{ steps.release.outputs.version }}' + python python/scripts/build_standard_distribution.py \ + --project build/python-release/models-standard \ + --outdir build/python-release/artifacts + + - name: Verify archives and offline installation + shell: bash + run: | + set -euo pipefail + python python/scripts/verify_distributions.py \ + --standard-dir build/python-release/artifacts \ + --standard-version '${{ steps.release.outputs.version }}' + python python/scripts/assemble_release.py \ + models-standard '${{ steps.release.outputs.version }}' \ + build/python-release/artifacts build/python-release/release + python -m venv build/python-release/venv + build/python-release/venv/bin/python -m pip install \ + --no-index --find-links build/python-release/release \ + radixor-models-standard + build/python-release/venv/bin/python -c \ + "from importlib import resources; assert resources.files('radixor_models_standard').joinpath('manifest.json').is_file()" + + - name: Prepare complete PEP 503 index candidate + shell: bash + run: | + set -euo pipefail + index_root='build/python-release/index/python/simple' + if [[ "${ACT:-false}" != 'true' ]] && git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then + git fetch origin gh-pages:refs/remotes/origin/gh-pages + git worktree add --detach build/python-release/pages origin/gh-pages + mkdir -p "$(dirname "${index_root}")" + if [[ -d build/python-release/pages/python/simple ]]; then + cp -R build/python-release/pages/python/simple "${index_root}" + fi + fi + python python/scripts/update_simple_index.py \ + --root "${index_root}" \ + --repository "${GITHUB_REPOSITORY}" \ + --package radixor-models-standard \ + --version '${{ steps.release.outputs.version }}' \ + --tag '${{ steps.release.outputs.tag }}' \ + --artifacts build/python-release/release + + - name: Upload verified release candidate + if: ${{ env.ACT != 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-models-standard-release-${{ steps.release.outputs.version }} + path: build/python-release/release/* + if-no-files-found: error + retention-days: 14 + + - name: Upload package-index candidate + if: ${{ env.ACT != 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-models-standard-index-${{ steps.release.outputs.version }} + path: build/python-release/index/python/simple + if-no-files-found: error + retention-days: 14 + + publish: + name: Publish immutable GitHub Release + if: github.event_name == 'push' + needs: build + runs-on: ubuntu-latest + environment: python-github-release + permissions: + contents: write + id-token: write + attestations: write + + steps: + - name: Download verified release candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-models-standard-release-${{ needs.build.outputs.version }} + path: release + + - name: Verify release inventory + shell: bash + run: | + set -euo pipefail + cd release + sha256sum --check SHA256SUMS + awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files + find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \ + -printf '%f\n' | LC_ALL=C sort > actual-files + diff -u expected-files actual-files + rm expected-files actual-files + + - name: Attest package artifacts + uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2 + with: + subject-path: | + release/*.whl + release/*.tar.gz + + - name: Create and publish draft release exactly once + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.build.outputs.tag }} + RELEASE_VERSION: ${{ needs.build.outputs.version }} + run: | + set -euo pipefail + if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Release already exists; refusing to replace its assets: ${RELEASE_TAG}" >&2 + exit 1 + fi + gh release create "${RELEASE_TAG}" \ + release/*.whl release/*.tar.gz release/SHA256SUMS \ + --verify-tag --draft --title "radixor-models-standard ${RELEASE_VERSION}" \ + --notes "Precompiled standard Radixor model distribution ${RELEASE_VERSION}." + gh release edit "${RELEASE_TAG}" --draft=false + + publish-index: + name: Publish Python package index + if: github.event_name == 'push' + needs: [build, publish] + runs-on: ubuntu-latest + environment: python-github-pages + permissions: + contents: write + + steps: + - name: Check out gh-pages only + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: gh-pages + path: pages + + - name: Download validated index candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-models-standard-index-${{ needs.build.outputs.version }} + path: candidate + + - name: Commit package index + shell: bash + run: | + set -euo pipefail + mkdir -p pages/python/simple + rsync -a --delete candidate/ pages/python/simple/ + cd pages + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add python/simple + git diff --cached --quiet && exit 0 + git commit -m 'Index radixor-models-standard ${{ needs.build.outputs.version }}' + git push origin HEAD:gh-pages diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml new file mode 100644 index 0000000..cebdd7a --- /dev/null +++ b/.github/workflows/python-release.yml @@ -0,0 +1,452 @@ +name: Python Native Release + +on: + push: + tags: + - 'python@*' + workflow_dispatch: + inputs: + version: + description: Stable distribution version to validate without publishing + required: true + default: '4.1.0' + type: string + +permissions: + contents: read + +concurrency: + group: github-python-pages + cancel-in-progress: false + +jobs: + prepare: + name: Prepare versioned sources and sdist + runs-on: ubuntu-latest + outputs: + version: ${{ steps.release.outputs.version }} + tag: ${{ steps.release.outputs.tag }} + + steps: + - name: Check out repository + if: ${{ env.ACT != 'true' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12.13' + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.88.0 + + - name: Select and validate release + id: release + shell: bash + env: + REQUESTED_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then + tag="${GITHUB_REF_NAME}" + else + tag="python@${REQUESTED_VERSION}" + fi + eval "$(./tools/parse-python-release-tag.sh "${tag}")" + [[ "${PYTHON_DISTRIBUTION}" == 'radixor' ]] + if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then + [[ "$(git rev-parse "${tag}^{commit}")" == "${GITHUB_SHA}" ]] + git merge-base --is-ancestor "${GITHUB_SHA}" origin/main + fi + printf 'version=%s\ntag=%s\n' "${PYTHON_VERSION}" "${tag}" >> "${GITHUB_OUTPUT}" + + - name: Install pinned source-build tools + run: >- + python -m pip install --disable-pip-version-check + maturin==1.14.1 setuptools==80.9.0 wheel==0.45.1 + + - name: Materialize versioned native source + shell: bash + run: | + set -euo pipefail + rm -rf build/python-release + python python/scripts/prepare_release_tree.py \ + native '${{ steps.release.outputs.version }}' \ + build/python-release/native-source + + - name: Build and verify source distributions + shell: bash + run: | + set -euo pipefail + mkdir -p build/python-release/compiler-wheel build/python-release/compiler-runtime \ + build/python-release/native-sdist build/python-release/models + maturin build --release --locked \ + --manifest-path build/python-release/native-source/Cargo.toml \ + --out build/python-release/compiler-wheel + python -c "from pathlib import Path; import zipfile; wheels=list(Path('build/python-release/compiler-wheel').glob('*.whl')); assert len(wheels) == 1; zipfile.ZipFile(wheels[0]).extractall('build/python-release/compiler-runtime')" + PYTHONPATH=build/python-release/compiler-runtime \ + python python/scripts/build_standard_models.py \ + --project build/python-release/models-source \ + --distribution-version 1.0.0 + maturin sdist --manifest-path build/python-release/native-source/Cargo.toml \ + --out build/python-release/native-sdist + python python/scripts/build_standard_distribution.py \ + --project build/python-release/models-source \ + --outdir build/python-release/models + python python/scripts/verify_distributions.py \ + --standard-dir build/python-release/models \ + --standard-version 1.0.0 + PYTHONPATH=python/scripts python -c \ + "from pathlib import Path; from verify_distributions import _verify_main_sdist; _verify_main_sdist(next(Path('build/python-release/native-sdist').glob('*.tar.gz')), '${{ steps.release.outputs.version }}')" + + - name: Upload versioned native source + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-source-${{ steps.release.outputs.version }} + path: | + build/python-release/native-source + !build/python-release/native-source/target/** + if-no-files-found: error + retention-days: 1 + + - name: Upload native sdist + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-sdist-${{ steps.release.outputs.version }} + path: build/python-release/native-sdist/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload verified model fixture + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-model-fixture-${{ steps.release.outputs.version }} + path: build/python-release/models/* + if-no-files-found: error + retention-days: 1 + + build-linux-x86-64: + name: Build Linux x86-64 wheel + needs: prepare + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12.13' + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.88.0 + + - name: Download versioned native source + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-source-${{ needs.prepare.outputs.version }} + path: build/python-release/native-source + + - name: Download model fixture + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-model-fixture-${{ needs.prepare.outputs.version }} + path: build/python-release/models + + - name: Select Linux build isolation + id: linux-isolation + shell: bash + run: | + if [[ "${ACT:-}" == 'true' ]]; then + echo 'manylinux=off' >> "${GITHUB_OUTPUT}" + echo 'container=' >> "${GITHUB_OUTPUT}" + else + echo 'manylinux=auto' >> "${GITHUB_OUTPUT}" + echo 'container=quay.io/pypa/manylinux2014_x86_64@sha256:0a42cb7e5f4ba6bbfb8d0a86d1aab0c8876ba9c3be16bd99360ae42bf010ec77' >> "${GITHUB_OUTPUT}" + fi + + - name: Build manylinux wheel + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1 + with: + command: build + target: x86_64 + manylinux: ${{ steps.linux-isolation.outputs.manylinux }} + container: ${{ steps.linux-isolation.outputs.container }} + maturin-version: v1.14.1 + rust-toolchain: 1.88.0 + working-directory: build/python-release/native-source + args: --release --locked --out ../wheel + + - name: Smoke-test wheel with standard models + shell: bash + run: | + set -euo pipefail + python -m pip install --no-index \ + --find-links build/python-release/wheel \ + --find-links build/python-release/models radixor + python -c "from radixor import Stemmer; assert Stemmer('en').stem('running') == 'run'" + + - name: Upload Linux x86-64 wheel + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-wheel-linux-x86-64-${{ needs.prepare.outputs.version }} + path: build/python-release/wheel/*.whl + if-no-files-found: error + retention-days: 14 + + build-platform-wheels: + name: Build ${{ matrix.name }} wheel + needs: prepare + strategy: + fail-fast: false + matrix: + include: + - name: Linux aarch64 + os: ubuntu-24.04-arm + target: aarch64 + manylinux: auto + container: quay.io/pypa/manylinux2014_aarch64@sha256:63bfa74be47f0277e998cb7c1b571b27664ac848bb356b0f4588438f930285dd + artifact: linux-aarch64 + - name: macOS universal2 + os: macos-14 + target: universal2-apple-darwin + manylinux: 'off' + container: '' + artifact: macos-universal2 + - name: Windows x86-64 + os: windows-2022 + target: x86_64-pc-windows-msvc + manylinux: 'off' + container: '' + artifact: windows-x86-64 + runs-on: ${{ matrix.os }} + + steps: + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12.13' + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.88.0 + + - name: Download versioned native source + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-source-${{ needs.prepare.outputs.version }} + path: build/python-release/native-source + + - name: Download model fixture + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-model-fixture-${{ needs.prepare.outputs.version }} + path: build/python-release/models + + - name: Build platform wheel + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1 + with: + command: build + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + container: ${{ matrix.container }} + maturin-version: v1.14.1 + rust-toolchain: 1.88.0 + working-directory: build/python-release/native-source + args: --release --locked --out ../wheel + + - name: Smoke-test wheel with standard models + shell: bash + run: | + set -euo pipefail + python -m pip install --no-index \ + --find-links build/python-release/wheel \ + --find-links build/python-release/models radixor + python -c "from radixor import Stemmer; assert Stemmer('en').stem('running') == 'run'" + + - name: Upload platform wheel + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-wheel-${{ matrix.artifact }}-${{ needs.prepare.outputs.version }} + path: build/python-release/wheel/*.whl + if-no-files-found: error + retention-days: 14 + + assemble: + name: Assemble verified release + needs: [prepare, build-linux-x86-64, build-platform-wheels] + runs-on: ubuntu-latest + + steps: + - name: Check out repository + if: ${{ env.ACT != 'true' }} + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12.13' + + - name: Download native artifacts only + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: native-wheel-* + path: build/python-release/artifacts + merge-multiple: true + + - name: Download native sdist + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-sdist-${{ needs.prepare.outputs.version }} + path: build/python-release/artifacts + + - name: Enforce release allowlist and checksums + run: >- + python python/scripts/assemble_release.py native + '${{ needs.prepare.outputs.version }}' + build/python-release/artifacts build/python-release/release + + - name: Prepare complete PEP 503 index candidate + shell: bash + run: | + set -euo pipefail + index_root='build/python-release/index/python/simple' + if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then + git fetch origin gh-pages:refs/remotes/origin/gh-pages + git worktree add --detach build/python-release/pages origin/gh-pages + mkdir -p "$(dirname "${index_root}")" + if [[ -d build/python-release/pages/python/simple ]]; then + cp -R build/python-release/pages/python/simple "${index_root}" + fi + fi + python python/scripts/update_simple_index.py \ + --root "${index_root}" \ + --repository "${GITHUB_REPOSITORY}" \ + --package radixor \ + --version '${{ needs.prepare.outputs.version }}' \ + --tag '${{ needs.prepare.outputs.tag }}' \ + --artifacts build/python-release/release + + - name: Upload verified release candidate + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-native-release-${{ needs.prepare.outputs.version }} + path: build/python-release/release/* + if-no-files-found: error + retention-days: 14 + + - name: Upload package-index candidate + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-native-index-${{ needs.prepare.outputs.version }} + path: build/python-release/index/python/simple + if-no-files-found: error + retention-days: 14 + + publish: + name: Publish immutable GitHub Release + if: github.event_name == 'push' + needs: [prepare, assemble] + runs-on: ubuntu-latest + environment: python-github-release + permissions: + contents: write + id-token: write + attestations: write + + steps: + - name: Download verified release candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-native-release-${{ needs.prepare.outputs.version }} + path: release + + - name: Verify release inventory + shell: bash + run: | + set -euo pipefail + cd release + sha256sum --check SHA256SUMS + awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files + find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \ + -printf '%f\n' | LC_ALL=C sort > actual-files + diff -u expected-files actual-files + rm expected-files actual-files + + - name: Require published standard models + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$(gh release view 'python-models-standard@1.0.0' --json isDraft --jq '.isDraft')" == 'false' ]] + + - name: Attest package artifacts + uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2 + with: + subject-path: | + release/*.whl + release/*.tar.gz + + - name: Create and publish draft release exactly once + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + RELEASE_VERSION: ${{ needs.prepare.outputs.version }} + run: | + set -euo pipefail + if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Release already exists; refusing to replace its assets: ${RELEASE_TAG}" >&2 + exit 1 + fi + gh release create "${RELEASE_TAG}" \ + release/*.whl release/*.tar.gz release/SHA256SUMS \ + --verify-tag --draft --title "radixor ${RELEASE_VERSION}" \ + --notes "Native Rust/Python Radixor distribution ${RELEASE_VERSION}." + gh release edit "${RELEASE_TAG}" --draft=false + + publish-index: + name: Publish Python package index + if: github.event_name == 'push' + needs: [prepare, assemble, publish] + runs-on: ubuntu-latest + environment: python-github-pages + permissions: + contents: write + + steps: + - name: Check out gh-pages only + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: gh-pages + path: pages + + - name: Download validated index candidate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-native-index-${{ needs.prepare.outputs.version }} + path: candidate + + - name: Commit package index + shell: bash + run: | + set -euo pipefail + mkdir -p pages/python/simple + rsync -a --delete candidate/ pages/python/simple/ + cd pages + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add python/simple + git diff --cached --quiet && exit 0 + git commit -m 'Index radixor ${{ needs.prepare.outputs.version }}' + git push origin HEAD:gh-pages diff --git a/.gitignore b/.gitignore index fff4645..74bbfe2 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,9 @@ local.properties # Typically, this file would be tracked if it contains build/dependency configurations: #.project +# Eclipse and Buildship create project descriptors during import. +**/.project + # PMD plugin conf .pmd @@ -109,3 +112,21 @@ gradle-app.setting # Gradle task-name cache .gradletasknamecache + +##---------------------------------------------------------------------------------------- Python tooling +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +.coverage.* +htmlcov/ + +# Workspace-local Rust/Python build and test scratch directories. +/.cargo-target/ +/pytest-of-*/ + +# tempfile.NamedTemporaryFile-style dictionary scratch files must not survive +# as repository candidates when a process is interrupted. +/tmp*.gz diff --git a/.project b/.project deleted file mode 100644 index 5da9344..0000000 --- a/.project +++ /dev/null @@ -1,22 +0,0 @@ - - - Radixor - - - - org.eclipse.jdt.core.javanature - org.eclipse.buildship.core.gradleprojectnature - - - - org.eclipse.jdt.core.javabuilder - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - - - diff --git a/README.md b/README.md index bec06d2..a693307 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,19 @@ -Radixor banner +

+ Radixor logo +

[![License](https://img.shields.io/github/license/leogalambos/Radixor)](LICENSE) [![Java](https://img.shields.io/badge/Java-21%2B-brightgreen)](#) +[![Python](https://img.shields.io/badge/Python-3.9%2B-1769ef)](docs/python/fast-track.md) [![Maven Central](https://img.shields.io/maven-central/v/org.egothor/radixor)](https://central.sonatype.com/artifact/org.egothor/radixor) [![Published reports](https://img.shields.io/badge/reports-GitHub%20Pages-blue)](https://leogalambos.github.io/Radixor/builds/latest/) [![Quality gates](https://github.com/leogalambos/Radixor/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/leogalambos/Radixor/actions/workflows/build.yml) [![Coverage](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/coverage/) [![Mutation score](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/pitest/) -*Deterministic, multi-language stemming for Java, built around compact dictionary-derived patch-command tries with an explicit quality/speed trade-off.* +*Deterministic, multi-language stemming for Java and Python, built around compact dictionary-trained patch-command tries with an explicit quality/speed trade-off.* -**Radixor** is a modern multi-language stemming toolkit for Java in the tradition of the original **Egothor** approach. It learns compact word-to-stem transformations from dictionary data, stores them in compiled patch-command tries, and exposes a runtime model designed for speed, determinism, and operational simplicity. Unlike a closed-form dictionary lookup stemmer, Radixor can also generalize beyond explicitly listed word forms. +**Radixor** is a modern multi-language stemming toolkit for Java and Python in the tradition of the original **Egothor** approach. It learns compact word-to-stem transformations from dictionary data, stores them in compiled patch-command tries, and exposes native runtime implementations designed for speed, determinism, and operational simplicity. Unlike a closed-form dictionary lookup stemmer, Radixor can also generalize beyond explicitly listed word forms. It is particularly well suited to systems that need stemming which is: @@ -22,7 +25,35 @@ It is particularly well suited to systems that need stemming which is: It also retains the operational advantages of a compiled artifact model: predictable runtime behavior, direct binary loading, and clear separation between preparation-time compilation and live request processing. -## Add Radixor and a model +## Choose a runtime + +For Python, one installation provides the native runtime and the separate +standard package of 20 precompiled models: + +From PyPI, once publication is enabled: + +```bash +python -m pip install --only-binary=:all: radixor +``` + +Or from the GitHub Releases-backed index: + +```bash +python -m pip install --only-binary=:all: \ + --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor +``` + +```python +from radixor import Stemmer + +english = Stemmer("en") +print(english.stemWord("running")) # run +``` + +Continue with the [Python Fast Track](docs/python/fast-track.md) or +[Python Quick Start](docs/python/quick-start.md). + +### Java dependencies The core artifact contains the algorithm and registry, but no language dictionary. Add either one minimal model or the optional standard default pack: @@ -87,16 +118,16 @@ Radixor performance is best read together with stemming quality. The English dic | Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token | | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 20.627 | 2.117 | 98.0 | -| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.713 | 2.104 | 103.2 | -| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 17.408 | 1.438 | 82.7 | -| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 16.946 | 1.531 | 80.5 | -| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 15.735 | 1.278 | 74.8 | -| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 14.714 | 1.089 | 69.9 | -| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.090 | 1.254 | 71.7 | -| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 13.773 | 1.071 | 65.4 | -| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 15.396 | 2.497 | 73.1 | -| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 16.970 | 2.847 | 80.6 | +| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 15.064 | 0.658 | 71.6 | +| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 17.798 | 2.161 | 84.6 | +| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 13.900 | 0.941 | 66.0 | +| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 14.809 | 1.376 | 70.3 | +| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 13.186 | 0.930 | 62.6 | +| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 12.852 | 0.943 | 61.1 | +| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 12.358 | 0.831 | 58.7 | +| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 11.657 | 0.921 | 55.4 | +| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 11.494 | 1.256 | 54.6 | +| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 9.895 | 0.925 | 47.0 | Column meanings: @@ -109,7 +140,7 @@ Column meanings: - `Error ms` is the JMH score error converted to milliseconds. - `ns/token` is average nanoseconds per changed token in that operation. -The contracted trie result is materially stronger than the older uncontracted profile: full English coverage reaches 97.478% all-token exactness and 97.197% changed-token exactness at 98.0 ns/token, while even a 10% deterministic dictionary slice remains at 92.868% all-token exactness and 76.516% changed-token exactness at 80.6 ns/token. This is why Radixor benchmark results are documented with both speed and quality instead of a single Porter speed badge. +The contracted trie result is materially stronger than the older uncontracted profile: full English coverage reaches 97.478% all-token exactness and 97.197% changed-token exactness at 71.6 ns/token, while even a 10% deterministic dictionary slice remains at 92.868% all-token exactness and 76.516% changed-token exactness at 47.0 ns/token. This is why Radixor benchmark results are documented with both speed and quality instead of a single Porter speed badge. For benchmark scope, workload design, environment, commands, report locations, and interpretation guidance, see [Benchmarking](docs/benchmarking.md). @@ -182,13 +213,22 @@ The repository keeps the front page concise and places detailed documentation un ### Getting Started -- [Fast Track](docs/fast-track.md) - The shortest path from adding core plus a model artifact to getting a first stem. +- [Python Fast Track](docs/python/fast-track.md) + The shortest path from `pip install` to the first native Python stem. -- [Quick Start](docs/quick-start.md) - A broader developer walkthrough covering loading options, querying, extension, persistence, and metadata. +- [Java Fast Track](docs/fast-track.md) + The shortest Java path from adding core plus a model artifact to getting a first stem. -- [Integration Deep Dive](docs/integration-deep-dive.md) +- [Python Quick Start](docs/python/quick-start.md) + Installation, standard models, batch use, PyStemmer migration, and deployment guidance. + +- [Java Quick Start](docs/quick-start.md) + A broader Java walkthrough covering loading options, querying, extension, persistence, and metadata. + +- [Python Overview](docs/python/index.md) + Runtime architecture, model packaging, API capabilities, and Java interoperability. + +- [Java Integration Deep Dive](docs/integration-deep-dive.md) Dependency setup, model selection, production lifecycle, search-pipeline guidance, and operational checklist. - [Built-in Languages](docs/built-in-languages.md) @@ -197,10 +237,29 @@ The repository keeps the front page concise and places detailed documentation un - [Dictionary Format](docs/dictionary-format.md) How to write and normalize stemming dictionaries. -- [Compilation (CLI tool)](docs/cli-compilation.md) - How to compile dictionaries into deployable binary artifacts. +- [Java CLI Compilation](docs/cli-compilation.md) + How to compile dictionaries into deployable binary artifacts from Java. -### Programmatic Usage +### Python + +The Python installation installs the native package together with the pure +`radixor-models-standard` 1.x distribution of the 2026.1 catalog: 20 precompiled v7 models, excluding +the optional PoliMorf model. Python runtime distributions contain no textual +dictionaries. + +- [Installation and Builds](docs/python/installation.md) + Wheels, source builds, Gradle tasks, host builds, and cross-compilation requirements. + +- [Usage and API](docs/python/usage.md) + Single and batch stemming, caching, custom dictionaries, and compiled models. + +- [Dictionary Compilation](docs/python/model-compilation.md) + Compile a textual dictionary once, load it directly, or share its version 7 binary with Java. + +- [Python Benchmarks](docs/python/performance.md) + Batch methodology and comparisons with available Python stemmers. + +### Java Programmatic Usage - [Programmatic Usage Overview](docs/programmatic-usage.md) Entry point to the Java API and the overall usage model. diff --git a/Radixor.png b/Radixor.png deleted file mode 100644 index 0aad67b..0000000 Binary files a/Radixor.png and /dev/null differ diff --git a/build.gradle b/build.gradle index 3d5e8fb..366c27b 100644 --- a/build.gradle +++ b/build.gradle @@ -762,7 +762,9 @@ tasks.register('prepareMkDocsSource', Sync) { buildsPage.setText('# Historical Builds\n\nThe Pages publication workflow replaces this staging placeholder with the retained build index.\n', 'UTF-8') File configuration = layout.buildDirectory.file('mkdocs/mkdocs.yml').get().asFile configuration.parentFile.mkdirs() - configuration.setText(layout.projectDirectory.file('mkdocs.yml').asFile.getText('UTF-8') + String configurationText = layout.projectDirectory.file('mkdocs.yml').asFile.getText('UTF-8') + .replace('custom_dir: docs/overrides', 'custom_dir: ../mkdocs-source/overrides') + configuration.setText(configurationText + '\ndocs_dir: ../mkdocs-source\nsite_dir: ../mkdocs-site\n', 'UTF-8') } } @@ -1133,6 +1135,7 @@ apply from: 'gradle/paicehusk-benchmarks.gradle' apply from: 'gradle/opennlp-benchmarks.gradle' apply from: 'gradle/hunspell-benchmarks.gradle' apply from: 'gradle/cistem-benchmarks.gradle' +apply from: 'gradle/python.gradle' gradle.taskGraph.whenReady { taskGraph -> def banner = """ diff --git a/docs/architecture-and-reduction.md b/docs/architecture-and-reduction.md index 09e45d4..c789034 100644 --- a/docs/architecture-and-reduction.md +++ b/docs/architecture-and-reduction.md @@ -13,11 +13,29 @@ Radixor does not keep a large flat table of final stems. Instead, it converts di The build-time flow is: -```text -Dictionary -> Mutable trie -> Reduced trie -> Compiled trie +```mermaid +flowchart TD + dictionary[Training dictionary] + mutable[Mutable trie] + reduced[Reduced trie] + compiled[Compiled trie] + + dictionary --> mutable --> reduced --> compiled ``` -For registered models, the dictionary is an independently versioned GZip resource discovered through a descriptor and verified before this flow begins. The model resource is input to trie construction, not a precompiled trie. See [Model Selection and Loading](model-selection-and-loading.md) for discovery and [Architecture](architecture.md) for component and release boundaries. +Both implementations follow this conceptual flow. Java materializes its +object-based compiled trie and exposes multiple reduction modes; the Python +extension implements the production dominant-result profile in Rust and stores +the runtime trie in flat arrays. Their persisted interoperability boundary is +the version 7 binary stream, not their in-memory representation. + +For registered Java models, the dictionary is an independently versioned GZip +resource discovered through a descriptor and verified before this flow begins. +For Python's standard models, this flow runs during package preparation and the +installed `radixor-models-standard` distribution already contains validated +compiled version 7 tries. See [Model Selection and Loading](model-selection-and-loading.md) +for Java discovery and [Architecture](architecture.md) for component and release +boundaries. Explicit descriptors and stable model IDs now use the same compiled-value path as language defaults. `loadCompiled(descriptor, ...)` and `loadCompiled(modelId, ...)` first build with serialized patch commands and then map those values to `CompiledPatchCommand` while preserving metadata, reduction semantics, and ranked `getAll` order. Very large inputs can have a high temporary construction peak; PoliMorf is verified in an isolated 6 GiB JVM rather than increasing ordinary test or Gradle daemon heaps. diff --git a/docs/architecture.md b/docs/architecture.md index 4829bb0..51c82df 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,7 +2,7 @@ This document explains the structural architecture of **Radixor**: what data is stored, how it flows through the build pipeline, and how runtime lookup works once a compiled trie has been produced. -## Component boundaries +## Java component boundaries | Component | Responsibility | |---|---| @@ -19,6 +19,23 @@ This document explains the structural architecture of **Radixor**: what data is Read [Model Selection and Loading](model-selection-and-loading.md) for executable application examples and [Stemmer Models](stemmer-models.md) for artifact maintenance. +## Python component boundaries + +The Python distribution is a separate native implementation rather than a JVM +wrapper. The `radixor` wheel contains the Rust/PyO3 runtime but no language +data. Its mandatory `radixor-models-standard` dependency supplies 20 validated, +precompiled, GZip-compressed version 7 `.rxc` tries. It does not use Java model +JARs, `ServiceLoader`, descriptors, or the Java registry. + +`Stemmer("")` resolves and synchronously loads a compiled standard model; +it does not parse a textual dictionary at application startup. +`radixor.compile(...)` remains available for application-owned textual +dictionaries, and `Stemmer(compiled=...)` loads the resulting version 7 +artifact. The Java and Python in-memory layouts are intentionally different; +the shared dictionary syntax and version 7 binary stream are their +interoperability boundaries. See [Radixor for Python](python/index.md) and +[Compiling Dictionaries in Python](python/model-compilation.md). + ## Runtime model discovery and loading The implemented sequence is: @@ -100,8 +117,14 @@ That matters because many words share similar transformation patterns. Once thos The full build-time flow is: -```text -Dictionary -> Mutable trie -> Reduced trie -> Compiled trie +```mermaid +flowchart TD + dictionary[Training dictionary] + mutable[Mutable trie] + reduced[Reduced trie] + compiled[Compiled trie] + + dictionary --> mutable --> reduced --> compiled ``` Each stage has a different purpose. diff --git a/docs/assets/data/homepage-performance.json b/docs/assets/data/homepage-performance.json new file mode 100644 index 0000000..ee7fb42 --- /dev/null +++ b/docs/assets/data/homepage-performance.json @@ -0,0 +1,50 @@ +{ + "source": "Python all-language batch benchmark, 2026-08-08", + "environment": { + "processor": "AMD Ryzen 5 5625U with Radeon Graphics", + "platform": "Linux-7.1.6-201.fc44.x86_64-x86_64-with-glibc2.43", + "python": "CPython 3.14.6", + "cpu_governor": "schedutil" + }, + "batch_size": 100, + "direct_pystemmer_comparisons": 18, + "direct_pystemmer_wins": 18, + "geometric_mean_speedup_vs_pystemmer": 1.665046904842523, + "maximum_speedup_vs_pystemmer": { + "language": "it", + "speedup": 3.027396185495421 + }, + "radixor_throughput_mwords_per_second": { + "minimum": { + "language": "ru", + "value": 3.6574304356731138 + }, + "maximum": { + "language": "pt", + "value": 5.99085795076713 + } + }, + "languages": { + "cs": {"radixor_ns_per_word": 224.3, "radixor_mwords_per_second": 4.46, "pystemmer_ns_per_word": 236.6, "speedup_vs_pystemmer": 1.05}, + "da": {"radixor_ns_per_word": 178.3, "radixor_mwords_per_second": 5.61, "pystemmer_ns_per_word": 267.6, "speedup_vs_pystemmer": 1.50}, + "de": {"radixor_ns_per_word": 230.9, "radixor_mwords_per_second": 4.33, "pystemmer_ns_per_word": 635.5, "speedup_vs_pystemmer": 2.75}, + "en": {"radixor_ns_per_word": 180.5, "radixor_mwords_per_second": 5.54, "pystemmer_ns_per_word": 331.9, "speedup_vs_pystemmer": 1.84}, + "es": {"radixor_ns_per_word": 184.2, "radixor_mwords_per_second": 5.43, "pystemmer_ns_per_word": 316.6, "speedup_vs_pystemmer": 1.72}, + "fa": {"radixor_ns_per_word": 210.1, "radixor_mwords_per_second": 4.76, "pystemmer_ns_per_word": 497.1, "speedup_vs_pystemmer": 2.37}, + "fi": {"radixor_ns_per_word": 227.8, "radixor_mwords_per_second": 4.39, "pystemmer_ns_per_word": 258.8, "speedup_vs_pystemmer": 1.14}, + "fr": {"radixor_ns_per_word": 234.2, "radixor_mwords_per_second": 4.27, "pystemmer_ns_per_word": 503.7, "speedup_vs_pystemmer": 2.15}, + "he": {"radixor_ns_per_word": 228.6, "radixor_mwords_per_second": 4.37, "pystemmer_ns_per_word": null, "speedup_vs_pystemmer": null}, + "hu": {"radixor_ns_per_word": 198.2, "radixor_mwords_per_second": 5.04, "pystemmer_ns_per_word": 264.7, "speedup_vs_pystemmer": 1.34}, + "it": {"radixor_ns_per_word": 170.8, "radixor_mwords_per_second": 5.86, "pystemmer_ns_per_word": 517.0, "speedup_vs_pystemmer": 3.03}, + "nb": {"radixor_ns_per_word": 187.1, "radixor_mwords_per_second": 5.34, "pystemmer_ns_per_word": 239.7, "speedup_vs_pystemmer": 1.28}, + "nl": {"radixor_ns_per_word": 187.1, "radixor_mwords_per_second": 5.35, "pystemmer_ns_per_word": 354.8, "speedup_vs_pystemmer": 1.90}, + "nn": {"radixor_ns_per_word": 168.7, "radixor_mwords_per_second": 5.93, "pystemmer_ns_per_word": 231.2, "speedup_vs_pystemmer": 1.37}, + "pl": {"radixor_ns_per_word": 194.6, "radixor_mwords_per_second": 5.14, "pystemmer_ns_per_word": 214.5, "speedup_vs_pystemmer": 1.10}, + "pt": {"radixor_ns_per_word": 166.9, "radixor_mwords_per_second": 5.99, "pystemmer_ns_per_word": 293.2, "speedup_vs_pystemmer": 1.76}, + "ru": {"radixor_ns_per_word": 273.4, "radixor_mwords_per_second": 3.66, "pystemmer_ns_per_word": 414.4, "speedup_vs_pystemmer": 1.52}, + "sv": {"radixor_ns_per_word": 189.3, "radixor_mwords_per_second": 5.28, "pystemmer_ns_per_word": 212.5, "speedup_vs_pystemmer": 1.12}, + "uk": {"radixor_ns_per_word": 221.5, "radixor_mwords_per_second": 4.51, "pystemmer_ns_per_word": null, "speedup_vs_pystemmer": null}, + "yi": {"radixor_ns_per_word": 227.5, "radixor_mwords_per_second": 4.39, "pystemmer_ns_per_word": 624.2, "speedup_vs_pystemmer": 2.74} + }, + "pystemmer_missing_for_radixor_languages": ["he", "uk"] +} diff --git a/docs/assets/images/banner.jpg b/docs/assets/images/banner.jpg deleted file mode 100644 index 7fe6599..0000000 Binary files a/docs/assets/images/banner.jpg and /dev/null differ diff --git a/docs/assets/images/flags/cs.svg b/docs/assets/images/flags/cs.svg new file mode 100644 index 0000000..88030ce --- /dev/null +++ b/docs/assets/images/flags/cs.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/da.svg b/docs/assets/images/flags/da.svg new file mode 100644 index 0000000..99b37fc --- /dev/null +++ b/docs/assets/images/flags/da.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/de.svg b/docs/assets/images/flags/de.svg new file mode 100644 index 0000000..30d1788 --- /dev/null +++ b/docs/assets/images/flags/de.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/en.svg b/docs/assets/images/flags/en.svg new file mode 100644 index 0000000..347a40e --- /dev/null +++ b/docs/assets/images/flags/en.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/es.svg b/docs/assets/images/flags/es.svg new file mode 100644 index 0000000..84e89ca --- /dev/null +++ b/docs/assets/images/flags/es.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/fa.svg b/docs/assets/images/flags/fa.svg new file mode 100644 index 0000000..4154636 --- /dev/null +++ b/docs/assets/images/flags/fa.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/fi.svg b/docs/assets/images/flags/fi.svg new file mode 100644 index 0000000..4667051 --- /dev/null +++ b/docs/assets/images/flags/fi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/fr.svg b/docs/assets/images/flags/fr.svg new file mode 100644 index 0000000..e4b1ee0 --- /dev/null +++ b/docs/assets/images/flags/fr.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/he.svg b/docs/assets/images/flags/he.svg new file mode 100644 index 0000000..2d7624f --- /dev/null +++ b/docs/assets/images/flags/he.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/hu.svg b/docs/assets/images/flags/hu.svg new file mode 100644 index 0000000..634d8df --- /dev/null +++ b/docs/assets/images/flags/hu.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/it.svg b/docs/assets/images/flags/it.svg new file mode 100644 index 0000000..8f5e3b8 --- /dev/null +++ b/docs/assets/images/flags/it.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/nb.svg b/docs/assets/images/flags/nb.svg new file mode 100644 index 0000000..80dcbaa --- /dev/null +++ b/docs/assets/images/flags/nb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/nl.svg b/docs/assets/images/flags/nl.svg new file mode 100644 index 0000000..4506b8b --- /dev/null +++ b/docs/assets/images/flags/nl.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/nn.svg b/docs/assets/images/flags/nn.svg new file mode 100644 index 0000000..80dcbaa --- /dev/null +++ b/docs/assets/images/flags/nn.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/pl.svg b/docs/assets/images/flags/pl.svg new file mode 100644 index 0000000..6646a4a --- /dev/null +++ b/docs/assets/images/flags/pl.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/pt.svg b/docs/assets/images/flags/pt.svg new file mode 100644 index 0000000..7ec82c5 --- /dev/null +++ b/docs/assets/images/flags/pt.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/ru.svg b/docs/assets/images/flags/ru.svg new file mode 100644 index 0000000..937a4d1 --- /dev/null +++ b/docs/assets/images/flags/ru.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/sv.svg b/docs/assets/images/flags/sv.svg new file mode 100644 index 0000000..b69c420 --- /dev/null +++ b/docs/assets/images/flags/sv.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/uk.svg b/docs/assets/images/flags/uk.svg new file mode 100644 index 0000000..2e25bb3 --- /dev/null +++ b/docs/assets/images/flags/uk.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/flags/yi.svg b/docs/assets/images/flags/yi.svg new file mode 100644 index 0000000..2d7624f --- /dev/null +++ b/docs/assets/images/flags/yi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/hero-lines.svg b/docs/assets/images/hero-lines.svg new file mode 100644 index 0000000..94f894f --- /dev/null +++ b/docs/assets/images/hero-lines.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/java-award.svg b/docs/assets/images/java-award.svg new file mode 100644 index 0000000..d455852 --- /dev/null +++ b/docs/assets/images/java-award.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/images/radixor-logo.png b/docs/assets/images/radixor-logo.png new file mode 100644 index 0000000..4f11b5b Binary files /dev/null and b/docs/assets/images/radixor-logo.png differ diff --git a/docs/assets/javascripts/landing-v2.js b/docs/assets/javascripts/landing-v2.js new file mode 100644 index 0000000..75fb00f --- /dev/null +++ b/docs/assets/javascripts/landing-v2.js @@ -0,0 +1 @@ +document.addEventListener("DOMContentLoaded",()=>{const b=document.querySelector(".rx2-menu"),n=document.querySelector(".rx2-nav");if(b&&n)b.addEventListener("click",()=>{const o=n.classList.toggle("open");b.setAttribute("aria-expanded",String(o));});document.querySelectorAll("[data-copy]").forEach(btn=>btn.addEventListener("click",async()=>{const el=document.getElementById(btn.dataset.copy);if(!el)return;try{await navigator.clipboard.writeText(el.innerText);const t=btn.textContent;btn.textContent="Copied";setTimeout(()=>btn.textContent=t,1200);}catch(e){}}));}); \ No newline at end of file diff --git a/docs/assets/javascripts/mermaid.js b/docs/assets/javascripts/mermaid.js new file mode 100644 index 0000000..09328de --- /dev/null +++ b/docs/assets/javascripts/mermaid.js @@ -0,0 +1,14 @@ +const renderRadixorDiagrams = () => { + mermaid.initialize({ + startOnLoad: false, + theme: "neutral", + flowchart: { htmlLabels: true, useMaxWidth: true }, + }); + return mermaid.run({ querySelector: ".mermaid" }); +}; + +if (typeof document$ === "undefined") { + document.addEventListener("DOMContentLoaded", renderRadixorDiagrams); +} else { + document$.subscribe(renderRadixorDiagrams); +} diff --git a/docs/assets/stylesheets/landing-v2.css b/docs/assets/stylesheets/landing-v2.css new file mode 100644 index 0000000..3dc4188 --- /dev/null +++ b/docs/assets/stylesheets/landing-v2.css @@ -0,0 +1,136 @@ + +:root{--rx2-navy:#0b1537;--rx2-text:#172554;--rx2-muted:#53627b;--rx2-line:#dce6f2;--rx2-purple:#6544ef;--rx2-blue:#1769ef;--rx2-cyan:#0a97cb;--rx2-teal:#0aa98f;--rx2-green:#079868;--rx2-red:#ef2f35;--rx2-shadow:0 12px 30px rgba(33,64,112,.085);--rx2-max:1050px} +.rx2-header{height:66px;background:#fff;border-bottom:1px solid #e2eaf3;position:relative;z-index:20}.rx2-header-inner{height:100%;width:min(var(--rx2-max),calc(100% - 42px));margin:auto;display:flex;align-items:center}.rx2-brand{display:flex;align-items:center;gap:10px;color:#11182e;text-decoration:none;font:850 1.23rem/1 system-ui,-apple-system,"Segoe UI",sans-serif;letter-spacing:.12em}.rx2-brand img{width:38px;height:38px}.rx2-nav{margin-left:auto;display:flex;gap:30px;align-items:center}.rx2-nav a{font:650 .74rem/1 system-ui,-apple-system,"Segoe UI",sans-serif;color:#11182e;text-decoration:none}.rx2-nav a:hover{color:#1769ef}.rx2-menu{display:none;margin-left:auto;width:38px;height:38px;border:1px solid var(--rx2-line);border-radius:10px;background:#fff;color:#172554} +.rx2-page{background:#fff;color:var(--rx2-navy);font-family:system-ui,-apple-system,"Segoe UI",Roboto,Arial,sans-serif;overflow:hidden}.rx2-page *{box-sizing:border-box}.rx2-page svg{display:block}.rx2-shell{width:min(var(--rx2-max),calc(100% - 42px));margin:0 auto}.rx2-hero{position:relative;padding:30px 0 18px;background:#fff}.rx2-hero:after{content:"";position:absolute;top:-55px;left:33%;width:820px;height:680px;background:url('../images/hero-lines.svg') center/contain no-repeat;opacity:.75;pointer-events:none;z-index:0} +.rx2-hero-top-grid{position:relative;z-index:1;display:grid;grid-template-columns:minmax(0,1.42fr) minmax(330px,.78fr);gap:72px;align-items:center;min-height:390px} +.rx2-hero-copy{padding:8px 0 16px 5px}.rx2-kicker{display:inline-flex;align-items:center;gap:8px;padding:6px 12px;border:1px solid #bfe8df;border-radius:999px;background:#f0fbf8;color:#087966;font-size:.68rem;font-weight:680}.rx2-kicker span{width:7px;height:7px;background:#099f7b;border-radius:50%}.rx2-hero h1{margin:22px 0 15px;max-width:650px;font-size:3.34rem;line-height:.99;letter-spacing:-.05em;font-weight:860;color:#0a1434}.rx2-hero h1>span{display:inline-block;margin-top:9px}.rx2-hero h1 b{font-weight:860;color:#6044ef}.rx2-hero h1 em{font-style:normal;font-weight:860;color:#0b9e8e}.rx2-hero-copy>p{max-width:650px;margin:0;color:#4b5b74;font-size:1rem;line-height:1.56} +.rx2-hero-actions{position:relative;padding:23px 24px 21px;border:1px solid #dce6f2;border-radius:17px;background:rgba(255,255,255,.94);box-shadow:0 13px 32px rgba(33,64,112,.07);backdrop-filter:blur(3px)}.rx2-action-eyebrow{margin-bottom:12px;color:#66748b;font-size:.61rem;font-weight:800;text-transform:uppercase;letter-spacing:.13em}.rx2-cta-row{display:grid;grid-template-columns:1fr;gap:11px;margin:0}.rx2-btn{height:52px;width:100%;padding:0 17px;border-radius:9px;color:#fff;text-decoration:none;display:inline-flex;align-items:center;gap:10px;font-size:.83rem;font-weight:740;box-shadow:0 9px 21px rgba(47,73,143,.14)}.rx2-btn b{margin-left:auto;font-size:1.25rem;font-weight:450}.rx2-java{background:linear-gradient(100deg,#5b36e8,#7553f1)}.rx2-python{background:linear-gradient(100deg,#1195ad,#08a58f)}.rx2-btn-icon{font-size:1.25rem}.rx2-python-glyph{display:grid;place-items:center;width:24px;height:24px;border-radius:8px;background:#fff;color:#138e9d;font-size:.68rem;font-weight:900} +.rx2-assurances{display:grid;grid-template-columns:1fr;gap:0;margin-top:17px;border-top:1px solid #e4ecf5;border-bottom:1px solid #e4ecf5}.rx2-assurances>div{display:grid;grid-template-columns:33px 1fr;align-items:center;gap:10px;min-height:61px;padding:8px 2px;color:#42516b;border-bottom:1px solid #edf2f7}.rx2-assurances>div:last-child{border-bottom:0}.rx2-assurances svg{width:31px;height:31px;flex:none;color:#2459ff}.rx2-assurances>div:nth-child(2) svg{color:#5d48ef}.rx2-assurances>div:nth-child(3) svg{color:#0a9c8b}.rx2-assurances span{display:flex;flex-direction:column;line-height:1.2}.rx2-assurances b{color:#1f2d49;font-size:.70rem}.rx2-assurances small{margin-top:3px;color:#66748b;font-size:.57rem} +.rx2-hero-links{display:flex;flex-wrap:wrap;gap:7px 14px;margin-top:14px}.rx2-hero-links a{color:#1769ef;text-decoration:none;font-size:.61rem;font-weight:720} +.rx2-highlight-section{position:relative;z-index:1;margin-top:17px}.rx2-highlight-card{border:1px solid #dce6f2;border-radius:17px;background:rgba(255,255,255,.98);box-shadow:0 12px 30px rgba(33,64,112,.075);padding:21px 24px 16px}.rx2-highlight-card h2{margin:0 0 14px;font-size:1.16rem;letter-spacing:-.025em} +.rx2-highlight-stats{display:grid;grid-template-columns:repeat(4,1fr);border:1px solid #e5edf6;border-radius:13px;overflow:hidden;background:#fbfdff}.rx2-highlight-stat{min-height:116px;padding:13px 10px 11px;text-align:center;border-right:1px solid #e5edf6;display:flex;flex-direction:column;align-items:center;justify-content:center}.rx2-highlight-stat:last-child{border-right:0}.rx2-highlight-stat i{display:block;width:34px;height:34px;margin-bottom:5px}.rx2-highlight-stat i svg{width:100%;height:100%}.rx2-highlight-stat strong{font-size:1.9rem;line-height:1.1;letter-spacing:-.04em}.rx2-highlight-stat span{margin-top:5px;color:#4b5b74;font-size:.65rem;line-height:1.26}.rx2-highlight-stat.purple{color:#663df0}.rx2-highlight-stat.teal{color:#0aa18c}.rx2-highlight-stat.blue{color:#155ff7} +.rx2-chart-title{margin-top:20px;color:#11182e;font-size:.93rem}.rx2-chart-title span{font-weight:600;font-size:.74rem}.rx2-throughput{margin-top:3px;color:#3d4a64;font-size:.79rem}.rx2-throughput b{font-size:1.72rem;background:linear-gradient(90deg,#155ff7,#0aa18c);color:transparent;background-clip:text;-webkit-background-clip:text}.rx2-chart-wrap{display:grid;grid-template-columns:34px 1fr;gap:7px;margin-top:9px}.rx2-ylabels{display:flex;flex-direction:column;justify-content:space-between;height:178px;padding:2px 0 13px;color:#66748b;font-size:.62rem}.rx2-chart{height:178px;display:flex;align-items:flex-end;gap:6px;padding:6px 2px 13px;border-bottom:1px solid #cad6e4;background:repeating-linear-gradient(to top,transparent 0 47px,rgba(209,220,233,.55) 48px 49px)}.rx2-bar{flex:1;min-width:8px;height:var(--h);border-radius:3px 3px 0 0;background:linear-gradient(180deg,#08bf9e 0%,#0ca8c0 45%,#2465ed 79%,#543cf0 100%)}.rx2-chart-range{display:flex;justify-content:space-between;padding:7px 3px 0 41px}.rx2-chart-range span{display:flex;flex-direction:column;color:#202b43;font-size:.73rem}.rx2-chart-range span:last-child{text-align:right}.rx2-chart-range small{font-size:.61rem;color:#526177}.rx2-note{border-top:1px solid #e5edf6;margin:12px 0 0;padding:11px 2px 0;color:#40506b;font-size:.64rem;line-height:1.42}.rx2-note b{color:#284df5} +.rx2-java-banner{margin-top:0;border:1px solid #d9e4f1;border-radius:15px;background:linear-gradient(90deg,#fbfdff,#fff);box-shadow:0 7px 18px rgba(35,73,129,.045);display:grid;grid-template-columns:150px 1.25fr 200px .95fr;min-height:160px;align-items:center;overflow:hidden}.rx2-java-banner>img{width:132px;height:125px;margin:auto}.rx2-java-copy{padding:12px 12px 12px 0}.rx2-java-copy h2{margin:0;font-size:1.65rem;line-height:1.04;letter-spacing:-.03em}.rx2-java-copy p{margin:10px 0 0;color:#4d5b72;font-size:.75rem;line-height:1.4}.rx2-win-box{height:116px;background:linear-gradient(145deg,#7b64f5,#5d42e6);color:#fff;border-radius:9px;display:flex;flex-direction:column;align-items:center;justify-content:center;box-shadow:0 12px 25px rgba(82,61,214,.17)}.rx2-win-box strong{font-size:2.52rem;line-height:1}.rx2-win-box span{font-size:.94rem;margin-top:4px}.rx2-win-box small{font-size:.61rem;margin-top:2px}.rx2-verify{padding:10px 18px;border-left:1px solid #e5edf6;color:#16213e;font-size:.69rem}.rx2-verify p{color:#53617a;line-height:1.42}.rx2-verify a{color:#1769ef;font-weight:700;text-decoration:none}.rx2-case-heading{text-align:center;font-size:.96rem;margin:16px 0 6px;color:#0d1734}.rx2-case-table{border:1px solid #dce6f2;border-radius:14px;overflow:hidden;background:#fff;box-shadow:0 5px 15px rgba(30,60,100,.035)}.rx2-case-head,.rx2-case-row{display:grid;grid-template-columns:1.05fr 1.15fr 1.15fr 1.3fr;align-items:center}.rx2-case-head{height:43px;border-bottom:1px solid #dfe8f2;font-size:.75rem;font-weight:700}.rx2-case-head>span,.rx2-case-row>span,.rx2-case-row>strong,.rx2-case-row>b{height:100%;display:flex;align-items:center;padding:7px 18px;border-left:1px solid #e6edf6}.rx2-case-head>span:first-child,.rx2-case-row>span:first-child{border-left:0}.rx2-case-head img{width:26px;height:26px;margin-right:8px}.rx2-case-head .snow{font-size:.76rem}.rx2-case-head .adv{color:#087f58}.rx2-case-head .adv svg{width:25px;height:25px;margin-right:7px}.rx2-case-row{min-height:42px;border-bottom:1px solid #e4ecf5;font-size:.75rem}.rx2-case-row:last-child{border-bottom:0}.rx2-case-row>span:first-child{display:grid;grid-template-columns:26px auto;grid-template-rows:auto auto;align-content:center;column-gap:8px}.rx2-case-row>span:first-child i{grid-row:1/3;width:24px;height:24px;color:#3154f6;font-style:normal;display:grid;place-items:center}.rx2-case-row>span:first-child i svg{width:24px;height:24px}.rx2-case-row small{font-size:.59rem;font-weight:500;color:#52617a}.rx2-case-row>strong{color:#079868;font-size:1rem}.rx2-case-row .red{color:#ef2f35;font-size:.95rem}.rx2-case-row .green{color:#078d64;font-size:.91rem;gap:5px}.rx2-case-row .green small{color:#078d64}.rx2-languages-strip{min-height:132px;margin-top:12px;border:1px solid #dce6f2;border-radius:13px;display:grid;grid-template-columns:190px 1fr;align-items:center;background:#fff;padding:10px 0}.rx2-language-label{display:flex;align-items:center;gap:12px;padding-left:20px}.rx2-language-label svg{width:38px;height:38px;color:#1769ef}.rx2-language-label strong,.rx2-language-label span{display:block}.rx2-language-label strong{font-size:1rem}.rx2-language-label span{font-size:.62rem;color:#5c6980}.rx2-language-list{display:grid;grid-template-columns:repeat(10,minmax(58px,1fr));align-items:start;gap:12px 6px;padding:7px 16px 7px 4px}.rx2-language{text-align:center;min-width:0}.rx2-language img{width:32px;height:32px;margin:auto;display:block}.rx2-language span{display:block;margin-top:4px;font-size:.51rem;line-height:1.16;color:#27334b}.rx2-code-grid{display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-top:10px}.rx2-code-card{border:1px solid #dce6f2;border-radius:13px;background:#fff;padding:10px 12px 9px;box-shadow:0 5px 13px rgba(26,54,99,.025)}.rx2-code-title{display:flex;align-items:center;gap:7px;height:27px}.rx2-code-title b{font-size:.94rem}.rx2-code-title button{margin-left:auto;border:1px solid #dbe5ef;background:#fff;color:#657188;border-radius:6px;font-size:.55rem;padding:3px 7px}.java-symbol{color:#e24924;font-size:1.2rem}.python-symbol{width:22px;height:22px;border-radius:6px;background:linear-gradient(135deg,#1769ef,#0aa98f);color:#fff;display:grid;place-items:center;font-size:.58rem;font-weight:900}.rx2-code-card pre{margin:5px 0 4px;min-height:124px;border:1px solid #e0e8f2;border-radius:7px;background:#f8fbff;padding:10px 11px;overflow:auto;font:500 .57rem/1.48 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;color:#203359;white-space:pre}.rx2-code-card a{font-size:.59rem;color:#1769ef;text-decoration:none;font-weight:700}.rx2-code-card .kw{color:#9a38a9}.rx2-code-card .type{color:#0b699a}.rx2-code-card .str{color:#168550}.rx2-benefits{min-height:78px;margin-top:10px;border:1px solid #dce6f2;border-radius:13px;background:#fff;display:grid;grid-template-columns:repeat(5,1fr);align-items:center;padding:7px 0}.rx2-benefits>div{min-height:57px;display:flex;gap:9px;align-items:flex-start;padding:8px 13px;border-right:1px solid #e4ecf5}.rx2-benefits>div:last-child{border-right:0}.rx2-benefits svg{width:28px;height:28px;flex:none;color:#3155f4}.rx2-benefits>div:nth-child(2) svg{color:#7752f5}.rx2-benefits>div:nth-child(3) svg{color:#2567ef}.rx2-benefits>div:nth-child(4) svg{color:#1467ee}.rx2-benefits>div:nth-child(5) svg{color:#5745ed}.rx2-benefits p{margin:0}.rx2-benefits b{display:block;font-size:.64rem}.rx2-benefits span{display:block;margin-top:3px;color:#657188;font-size:.52rem;line-height:1.35}.rx2-final-card{min-height:89px;margin:10px 0 24px;border:1px solid #dce6f2;border-radius:13px;background:linear-gradient(95deg,#fbfdff,#f4f8ff 55%,#f2fffc);display:grid;grid-template-columns:62px 1fr auto auto;gap:15px;align-items:center;padding:12px 18px}.rx2-final-card>img{width:49px;height:49px}.rx2-final-card h2{margin:0;font-size:1.08rem}.rx2-final-card p{margin:4px 0 0;color:#5a6780;font-size:.62rem}.rx2-final-card a{text-decoration:none;border-radius:7px;height:42px;padding:0 20px;display:flex;align-items:center;font-size:.71rem;font-weight:700}.rx2-gh{border:1px solid #cad7e6;color:#14203b;background:#fff}.rx2-doc{background:#0e5be8;color:#fff}.rx2-footer{height:62px;border-top:1px solid #e1e9f2;background:#fbfdff;color:#637086;font:550 .58rem/1 system-ui,-apple-system,"Segoe UI",sans-serif}.rx2-footer>.rx2-shell{height:100%;display:flex;align-items:center;justify-content:space-between}.rx2-footer a{color:#53617a;text-decoration:none;font-weight:650} +@media(max-width:900px){.rx2-header-inner,.rx2-shell{width:min(100% - 26px,var(--rx2-max))}.rx2-menu{display:block}.rx2-nav{display:none;position:absolute;top:61px;left:13px;right:13px;background:#fff;border:1px solid var(--rx2-line);border-radius:12px;box-shadow:var(--rx2-shadow);padding:16px;flex-direction:column;align-items:flex-start;gap:15px}.rx2-nav.open{display:flex}.rx2-hero-grid{grid-template-columns:1fr}.rx2-hero-copy{padding-top:0}.rx2-hero h1{max-width:670px}.rx2-highlight-card{max-width:620px}.rx2-java-banner{grid-template-columns:110px 1fr 170px}.rx2-verify{grid-column:1/-1;border-left:0;border-top:1px solid #e5edf6}.rx2-language-list{grid-template-columns:repeat(5,minmax(70px,1fr));padding:10px 14px}.rx2-languages-strip{grid-template-columns:170px 1fr}.rx2-benefits{grid-template-columns:repeat(2,1fr)}.rx2-benefits>div{border-bottom:1px solid #e4ecf5}.rx2-final-card{grid-template-columns:52px 1fr}.rx2-final-card a{grid-row:2}} +@media(max-width:650px){.rx2-hero{padding-top:16px}.rx2-hero h1{font-size:2.35rem}.rx2-cta-row{flex-direction:column}.rx2-btn{width:100%}.rx2-assurances{grid-template-columns:1fr}.rx2-highlight-stats{grid-template-columns:repeat(2,1fr)}.rx2-highlight-stat{border-bottom:1px solid #e5edf6}.rx2-java-banner{grid-template-columns:1fr;padding:14px}.rx2-java-banner>img{width:104px}.rx2-win-box{width:100%}.rx2-verify{padding:15px 0 4px}.rx2-case-head,.rx2-case-row{grid-template-columns:1.25fr 1fr 1fr}.rx2-case-head>*:last-child,.rx2-case-row>*:last-child{grid-column:1/-1;border-left:0;border-top:1px dashed #e5edf6}.rx2-languages-strip{grid-template-columns:1fr}.rx2-language-label{padding:14px 16px 6px}.rx2-language-list{grid-template-columns:repeat(4,minmax(62px,1fr));padding:8px 12px 14px}.rx2-code-grid{grid-template-columns:1fr}.rx2-benefits{grid-template-columns:1fr}.rx2-benefits>div{border-right:0}.rx2-final-card{grid-template-columns:1fr}.rx2-final-card a{grid-row:auto}.rx2-footer{height:auto;padding:16px 0}.rx2-footer>.rx2-shell{gap:10px;flex-direction:column;align-items:flex-start}} + + +/* Benchmark-linked language badges */ +.rx2-language { + text-decoration: none; + border-radius: 10px; + padding: 5px 2px 6px; + transition: transform 140ms ease, background-color 140ms ease, box-shadow 140ms ease; +} +.rx2-language:hover, +.rx2-language:focus-visible { + background: #f4f8ff; + box-shadow: 0 5px 16px rgba(38, 76, 145, .09); + transform: translateY(-2px); + outline: none; +} +.rx2-language:hover span, +.rx2-language:focus-visible span { + color: #145ff5; +} +.rx2-case-note { + margin: -2px 10px 13px; + color: #65738a; + font-size: .58rem; + line-height: 1.45; + text-align: right; +} +@media (max-width: 650px) { + .rx2-case-note { + margin: 4px 4px 12px; + text-align: left; + font-size: .62rem; + } +} + +/* Current Radixor mark and language-labelled performance chart. */ +.rx2-brand img, +.rx2-final-card > img, +.rx2-case-head img { + object-fit: cover; + border-radius: 22%; + box-shadow: 0 3px 10px rgba(24, 82, 176, .16); +} +.rx2-chart-wrap { + margin-top: 10px; +} +.rx2-ylabels, +.rx2-chart { + height: 206px; + padding-bottom: 38px; +} +.rx2-chart { + gap: clamp(3px, .65vw, 7px); + overflow: visible; + background: repeating-linear-gradient(to top, transparent 0 47px, rgba(209, 220, 233, .55) 48px 49px); +} +.rx2-bar { + position: relative; + min-width: 12px; +} +.rx2-bar img { + position: absolute; + left: 50%; + bottom: -32px; + width: 24px; + height: 24px; + max-width: none; + padding: 2px; + object-fit: cover; + border: 1px solid #cbd8e7; + border-radius: 50%; + background: #fff; + box-shadow: 0 2px 6px rgba(33, 64, 112, .14); + transform: translateX(-50%); +} +.rx2-bar:focus-visible { + outline: 2px solid #1769ef; + outline-offset: 2px; +} + +@media (max-width: 650px) { + .rx2-highlight-card { + padding-right: 14px; + padding-left: 14px; + } + .rx2-chart-wrap { + grid-template-columns: 27px minmax(360px, 1fr); + overflow-x: auto; + padding-bottom: 4px; + } + .rx2-ylabels { + position: sticky; + left: 0; + z-index: 2; + background: linear-gradient(90deg, #fff 80%, rgba(255, 255, 255, 0)); + } + .rx2-chart { + gap: 5px; + } + .rx2-bar { + min-width: 12px; + } + .rx2-bar img { + width: 17px; + height: 17px; + padding: 1px; + bottom: -26px; + } +} + +@media (max-width: 900px) { + .rx2-hero-top-grid { + grid-template-columns: minmax(0, 1fr); + gap: 22px; + min-height: 0; + } + .rx2-hero-copy, + .rx2-hero-copy > p { + max-width: 100%; + } +} diff --git a/docs/assets/stylesheets/radixor-docs-safety.css b/docs/assets/stylesheets/radixor-docs-safety.css new file mode 100644 index 0000000..cfd3166 --- /dev/null +++ b/docs/assets/stylesheets/radixor-docs-safety.css @@ -0,0 +1,146 @@ +/* + * Radixor documentation safety constraints. + * + * The landing page uses custom page chrome while standard documentation pages + * use Material for MkDocs. These size constraints ensure that a documentation + * logo can never inherit unconstrained intrinsic dimensions. The remaining + * rules carry the landing page's light blue/teal visual language into the + * standard Material documentation shell. + */ +:root, +[data-md-color-scheme="default"] { + --md-primary-fg-color: #ffffff; + --md-primary-fg-color--light: #ffffff; + --md-primary-fg-color--dark: #f7faff; + --md-primary-bg-color: #11182e; + --md-primary-bg-color--light: #4b5b74; + --md-accent-fg-color: #1769ef; + --md-accent-fg-color--transparent: rgba(23, 105, 239, 0.1); + --md-typeset-a-color: #0e62df; + --rx-doc-line: #dce6f2; + --rx-doc-soft: #f7faff; + --rx-doc-text: #172554; + --rx-doc-muted: #53627b; + --rx-doc-teal: #078d7a; + --rx-doc-purple: #6544ef; +} + +.md-header { + color: var(--rx-doc-text); + background: rgba(255, 255, 255, 0.97); + border-bottom: 1px solid var(--rx-doc-line); + box-shadow: 0 5px 18px rgba(33, 64, 112, 0.06); + backdrop-filter: blur(8px); +} + +.md-header__title, +.md-header__button, +.md-header__topic, +.md-source { + color: var(--rx-doc-text); +} + +.md-search__form { + background: #f3f7fc; + border: 1px solid #d8e4f0; +} + +.md-search__input, +.md-search__icon { + color: var(--rx-doc-text); +} + +.md-search__input::placeholder { + color: #69778d; +} + +.md-tabs { + color: var(--rx-doc-text); + background: #fbfdff; + border-bottom: 1px solid var(--rx-doc-line); +} + +.md-nav__link--active, +.md-nav__link:focus, +.md-nav__link:hover { + color: #145ff5; +} + +.md-nav__item--section > .md-nav__link { + color: #263653; +} + +.md-typeset h1, +.md-typeset h2, +.md-typeset h3 { + color: #0b1537; +} + +.md-typeset a:hover { + color: var(--rx-doc-purple); +} + +.md-typeset code { + border-radius: 0.22rem; + background: #f3f7fc; +} + +.md-typeset .admonition, +.md-typeset details { + border-color: #8bb6f2; + box-shadow: 0 3px 12px rgba(33, 64, 112, 0.06); +} + +.md-typeset table:not([class]) { + border-color: var(--rx-doc-line); + box-shadow: 0 2px 10px rgba(33, 64, 112, 0.04); +} + +.md-footer { + --md-footer-fg-color: var(--rx-doc-text); + --md-footer-fg-color--light: #31516f; + --md-footer-fg-color--lighter: var(--rx-doc-muted); + --md-footer-bg-color: #fbfdff; + --md-footer-bg-color--dark: #f5f9fe; + color: var(--rx-doc-muted); + background: #fbfdff; + border-top: 1px solid var(--rx-doc-line); +} + +.md-footer-meta { + color: var(--rx-doc-muted); + background: #f5f9fe; +} + +.md-footer a, +.md-footer-meta a { + color: #31516f; +} + +.md-header__button.md-logo img, +.md-header__button.md-logo svg, +.md-nav__button.md-logo img, +.md-nav__button.md-logo svg { + width: 1.6rem; + height: 1.6rem; + max-width: 1.6rem; + max-height: 1.6rem; + object-fit: cover; + border-radius: 0.38rem; + box-shadow: 0 2px 8px rgba(24, 82, 176, 0.18); +} + +.md-typeset .mermaid { + width: 100%; + max-width: 100%; + overflow-x: auto; + text-align: center; +} + +.md-typeset .mermaid svg { + display: block; + width: auto; + max-width: 100%; + height: auto; + margin-inline: auto; +} diff --git a/docs/benchmarks/data/stemming-quality.csv b/docs/benchmarks/data/stemming-quality.csv index 89675bc..e4febe8 100644 --- a/docs/benchmarks/data/stemming-quality.csv +++ b/docs/benchmarks/data/stemming-quality.csv @@ -35,8 +35,8 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S "ENGLISH_RADIXOR","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","ALL_CANDIDATES","374384","568441","228735","555084","13357","1355","584042","374506","311382","15","0","161561989623","15","161561989638","0.000000","0","311382","0.000000","0.999951829979","1.000000000000","0.999999999907","0.999999999907","0.999999999954","0.999961463612","0.999975914409","0.999990365625","0.999951829979","0.999975914699","0.999975914653","0.000000000093","","","","","" "ENGLISH_SNOWBALL_ORIGINAL_PORTER","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","321092","284940","360538","28415","175199063592","360538","175199424130","0.000206","28415","313355","9.067990","0.441440296958","0.909320100206","0.999997942128","0.999997779945","0.954659021167","0.492078968883","0.594347503684","0.750277266077","0.422826769235","0.633569676567","0.633568843266","0.000002220055","","","","","" "ENGLISH_SNOWBALL_ORIGINAL_PORTER","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","299877","283312","357325","28070","161561632313","357325","161561989638","0.000221","28070","311382","9.014651","0.442234838138","0.909853491852","0.999997788310","0.999997614573","0.954925640081","0.492899966248","0.595181398691","0.751026553880","0.423671353822","0.634325556555","0.634324660984","0.000002385427","","","","","" -"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","318385","284971","371381","28384","175199052749","371381","175199424130","0.000212","28384","313355","9.058097","0.434174040759","0.909419029535","0.999997880238","0.999997718233","0.954708454887","0.484848557029","0.587746607996","0.746086443827","0.416176453407","0.628367833992","0.628366984426","0.000002281767","","","","","" -"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","297220","283368","368027","28014","161561621611","368027","161561989638","0.000228","28014","311382","8.996666","0.435017155489","0.910033335260","0.999997722069","0.999997548679","0.955015528665","0.485724531207","0.588647215295","0.746914872138","0.417080138768","0.629190045142","0.629189132274","0.000002451321","","","","","" +"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","ALL_WORDS","PRIMARY_OUTPUT","396939","591946","250964","591946","0","1","591946","318389","284986","371197","28369","175199052933","371197","175199424130","0.000212","28369","313355","9.053310","0.434308721805","0.909466898566","0.999997881289","0.999997719369","0.954732389927","0.484985638615","0.587880000578","0.746191747709","0.416310229172","0.628481826499","0.628480977278","0.000002280631","","","","","" +"ENGLISH_SNOWBALL_PORTER2","US_UK","us-uk-default","1.0.0","8c79122993499e437ea8b54b620832dca29019298f281c1f3132f4d1be885460","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","374384","568441","228735","568441","0","1","568441","297224","283383","367843","27999","161561621795","367843","161561989638","0.000228","27999","311382","8.991849","0.435153080497","0.910081507602","0.999997723208","0.999997549911","0.955039615405","0.485862840613","0.588781726310","0.747020963182","0.417215208510","0.629304990872","0.629304078379","0.000002450089","","","","","" "FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","439975","12317229","1508153","19148370","1599840231184","1508153","1599841739337","0.000094","19148370","31465599","60.854936","0.890914189568","0.391450644242","0.999999057311","0.999987088650","0.695724850776","0.709786610775","0.543915310644","0.440884276934","0.373546480243","0.590549687554","0.590545009664","0.000012911350","","","","","" "FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","431848","11954192","1155011","18806691","1504704980042","1155011","1504706135053","0.000077","18806691","30760883","61.138333","0.911893118140","0.388616672675","0.999999232401","0.999986734091","0.694307952538","0.718420864905","0.544981470973","0.438999334093","0.374552942180","0.595295615141","0.595290947645","0.000013265909","","","","","" "FINNISH_RADIXOR","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","69091","30511413","804","954186","1599841738533","804","1599841739337","0.000000","954186","31465599","3.032474","0.999973649899","0.969675263452","0.999999999497","0.999999403084","0.984837631475","0.993763441201","0.984591422195","0.975587162604","0.969650487220","0.984707932542","0.984707638627","0.000000596916","","","","","" @@ -205,24 +205,26 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S "RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","37297","758584","10","758584","0","1","758584","37282","12780071","0","255156","287711428009","0","287711428009","0.000000","255156","13035227","1.957434","1.000000000000","0.980425657336","1.000000000000","0.999999113193","0.990212828668","0.996022851412","0.990116093179","0.984278980143","0.980425657336","0.990164459742","0.990164020680","0.000000886807","","","","","" "RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","ANY_CANDIDATE","37297","758584","10","749142","9442","4","768163","37306","","","","","0","287711428009","0.000000","0","13035227","0.000000","","","","","","","","","","","","","","","","","" "RUSSIAN_RADIXOR","RU_RU","ru-ru-default","1.0.0","df7ea25e63a875eeec7a4185be685bd5372a3c568db85c34c44fdf5d8d980a40","LOWERCASE_GROUPS_ONLY","ALL_CANDIDATES","37297","758584","10","749142","9442","4","768163","37306","13035227","0","0","287711428009","0","287711428009","0.000000","0","13035227","0.000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","1.000000000000","0.000000000000","","","","","" -"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5553","78545","4795","11150","389682670","4795","389687465","0.001230","11150","89695","12.431016","0.942464602832","0.875689837784","0.999987695268","0.999959092010","0.937838766526","0.928307194100","0.907851012801","0.888276938388","0.831251984337","0.908463909669","0.908443737019","0.000040907990","","","","","" -"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5539","78440","4795","11100","388399540","4795","388404335","0.001235","11100","89540","12.396694","0.942392022587","0.876033057851","0.999987654618","0.999959085584","0.938010356234","0.928327968188","0.908001736362","0.888546539947","0.831504743732","0.908606936602","0.908586757624","0.000040914416","","","","","" +"SNOWBALL_CZECH_DIRECT","CS_CZ","cs-cz-default","1.0.0","62afdaa6dc7a721b54a0dc278a0c648a63ad52a34a412d27b5b52fbcde9c1ce4","ALL_WORDS","PRIMARY_OUTPUT","5113","51401","2","51401","0","1","51401","10932","172114","11935","128395","1320693256","11935","1320705191","0.000904","128395","300509","42.725842","0.935153138566","0.572741581783","0.999990963161","0.999893770330","0.786366272472","0.830101137739","0.710395865923","0.620863799839","0.550863514742","0.731847721723","0.731803909891","0.000106229670","","","","","" +"SNOWBALL_CZECH_DIRECT","CS_CZ","cs-cz-default","1.0.0","62afdaa6dc7a721b54a0dc278a0c648a63ad52a34a412d27b5b52fbcde9c1ce4","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","5038","50697","2","50697","0","1","50697","10817","169464","11863","128023","1284758206","11863","1284770069","0.000923","128023","297487","43.034822","0.934576759115","0.569651783103","0.999990766441","0.999891145022","0.784821274772","0.828435805807","0.707848976847","0.617906692677","0.547806691450","0.729646021901","0.729601212980","0.000108854978","","","","","" +"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5409","79378","4816","10317","389682649","4816","389687465","0.001236","10317","89695","11.502313","0.942798774259","0.884976866046","0.999987641378","0.999961175252","0.942482253712","0.930637722143","0.912973218547","0.895966806178","0.839881072045","0.913430404878","0.913411201871","0.000038824748","","","","","" +"SNOWBALL_DANISH_DIRECT","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5403","79223","4816","10317","388399519","4816","388404335","0.001240","10317","89540","11.522225","0.942693273361","0.884777752960","0.999987600550","0.999961047005","0.942382676755","0.930511444787","0.912817794779","0.895784477125","0.839618042308","0.913276538697","0.913257272617","0.000038952995","","","","","" "SNOWBALL_DANISH_LUCENE_FILTER","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","ALL_WORDS","PRIMARY_OUTPUT","4179","27921","32","27921","0","1","27921","5546","78557","4961","11138","389682504","4961","389687465","0.001273","11138","89695","12.417638","0.940599631217","0.875823624505","0.999987269285","0.999958696913","0.937905446895","0.926889068757","0.907056629699","0.888055112164","0.829920977011","0.907633945058","0.907613558620","0.000041303087","","","","","" "SNOWBALL_DANISH_LUCENE_FILTER","DA_DK","da-dk-default","1.0.0","3f7b670a0e7b872bda0381f5154ce058a4656297b39b7157b4ccf6560257cb90","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4173","27875","32","27875","0","1","27875","5539","78440","4795","11100","388399540","4795","388404335","0.001235","11100","89540","12.396694","0.942392022587","0.876033057851","0.999987654618","0.999959085584","0.938010356234","0.928327968188","0.908001736362","0.888546539947","0.831504743732","0.908606936602","0.908586757624","0.000040914416","","","","","" "SNOWBALL_DUTCH_DIRECT","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","ALL_WORDS","PRIMARY_OUTPUT","4992","26201","85","26201","0","1","26201","12051","29267","2987","35170","343165676","2987","343168663","0.000870","35170","64437","54.580443","0.907391331308","0.454195570868","0.999991295825","0.999888830652","0.727093433346","0.756436964017","0.605371751249","0.504599968276","0.434073920266","0.641975952605","0.641933549660","0.000111169348","","","","","" "SNOWBALL_DUTCH_DIRECT","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4796","25402","84","25402","0","1","25402","11466","29053","2987","33965","322552096","2987","322555083","0.000926","33965","63018","53.897299","0.906772784020","0.461027008156","0.999990739566","0.999885462099","0.730508873861","0.759841613575","0.611268909508","0.511294841471","0.440163623968","0.646565343716","0.646521311443","0.000114537901","","","","","" "SNOWBALL_DUTCH_LUCENE_FILTER","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","ALL_WORDS","PRIMARY_OUTPUT","4992","26201","85","26201","0","1","26201","14573","15204","759","49233","343167904","759","343168663","0.000221","49233","64437","76.404861","0.952452546514","0.235951394385","0.999997788260","0.999854349712","0.617974591322","0.592568341791","0.378208955224","0.277738198319","0.233204491073","0.474059602198","0.474021680915","0.000145650288","","","","","" "SNOWBALL_DUTCH_LUCENE_FILTER","NL_NL","nl-nl-default","1.0.0","c098034adc42da2ca3e419160e6dd2c2b3868f8af334303b3a191e09caadaf5e","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4796","25402","84","25402","0","1","25402","14116","14874","715","48144","322554368","715","322555083","0.000222","48144","63018","76.397220","0.954134325486","0.236027801581","0.999997783324","0.999848554685","0.618012792452","0.593185189912","0.378439579172","0.277851461363","0.233379881694","0.474554767395","0.474515425112","0.000151445315","","","","","" -"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","381483","15082807","952306","16382792","1599840787031","952306","1599841739337","0.000060","16382792","31465599","52.065724","0.940611207417","0.479342757784","0.999999404750","0.999989164705","0.739671081267","0.788799811426","0.635056038739","0.531468350160","0.465261620083","0.671472389727","0.671468317739","0.000010835295","","","","","" -"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","372232","14663371","936765","16097512","1504705198288","936765","1504706135053","0.000062","16097512","30760883","52.331112","0.939951485038","0.476688884386","0.999999377443","0.999988679564","0.738344130915","0.786987247415","0.632573283171","0.528815026735","0.462601231486","0.669376145960","0.669371899615","0.000011320436","","","","","" +"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","381016","15095314","952479","16370285","1599840786858","952479","1599841739337","0.000060","16370285","31465599","52.025976","0.940647352567","0.479740239491","0.999999404642","0.999989172414","0.739869822067","0.789035310423","0.635413022080","0.531861528280","0.465644940456","0.671763638639","0.671759568086","0.000010827586","","","","","" +"SNOWBALL_FINNISH_DIRECT","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","371779","14675605","936938","16085278","1504705198115","936938","1504706135053","0.000062","16085278","30760883","52.291340","0.939988123652","0.477086597287","0.999999377328","0.999988687580","0.738542987307","0.787224487482","0.632931670824","0.529208871663","0.462984663835","0.669668377186","0.669664132316","0.000011312420","","","","","" "SNOWBALL_FINNISH_LUCENE_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","ALL_WORDS","PRIMARY_OUTPUT","57027","1788784","292","1788784","0","1","1788784","377778","15121052","1288634","16344547","1599840450703","1288634","1599841739337","0.000081","16344547","31465599","51.944179","0.921471136011","0.480558212161","0.999999194524","0.999988978388","0.740278703342","0.778598131291","0.631685095974","0.531413183368","0.461651842069","0.665447610018","0.665443363449","0.000011021612","","","","","" "SNOWBALL_FINNISH_LUCENE_FILTER","FI_FI","fi-fi-default","1.0.0","ca2628b3db31fee92f1b612ebbbd5e956a6dbbfb10e721325e55ef528f26072f","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","54762","1734784","274","1734784","0","1","1734784","372232","14663371","936765","16097512","1504705198288","936765","1504706135053","0.000062","16097512","30760883","52.331112","0.939951485038","0.476688884386","0.999999377443","0.999988679564","0.738344130915","0.786987247415","0.632573283171","0.528815026735","0.462601231486","0.669376145960","0.669371899615","0.000011320436","","","","","" "SNOWBALL_FRENCH_DIRECT","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","ALL_WORDS","PRIMARY_OUTPUT","59240","404011","2301","404011","0","1","404011","85627","3744838","1092238","1625361","81605779618","1092238","81606871856","0.001338","1625361","5370199","30.266309","0.774194575401","0.697336914330","0.999986615858","0.999966701086","0.848661765094","0.757496924470","0.733758618240","0.711462917671","0.579477680015","0.734761496202","0.734744993787","0.000033298914","","","","","" "SNOWBALL_FRENCH_DIRECT","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","57698","400712","2133","400712","0","1","400712","84526","3736871","1088903","1619380","80278407962","1088903","80279496865","0.001356","1619380","5356251","30.233460","0.774356818202","0.697665400669","0.999986436101","0.999966266576","0.848825918385","0.757698693319","0.734013322497","0.711763857056","0.579795455624","0.735011537210","0.734994818860","0.000033733424","","","","","" "SNOWBALL_FRENCH_LUCENE_FILTER","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","ALL_WORDS","PRIMARY_OUTPUT","59240","404011","2301","404011","0","1","404011","85202","3742072","1097843","1628127","81605774013","1097843","81606871856","0.001345","1628127","5370199","30.317815","0.773168950281","0.696821849619","0.999986547175","0.999966598516","0.848404198397","0.756589837411","0.733012775372","0.710860736247","0.578547882033","0.734003418250","0.733986862867","0.000033401484","","","","","" "SNOWBALL_FRENCH_LUCENE_FILTER","FR_FR","fr-fr-default","1.0.0","a988658758952fd599dc7360e0234178a6d65ac46e5cedc7dcd325a7cb7e71d9","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","57698","400712","2133","400712","0","1","400712","84810","3734232","1086494","1622019","80278410371","1086494","80279496865","0.001353","1622019","5356251","30.282729","0.774620254294","0.697172705312","0.999986466109","0.999966263711","0.848579585710","0.757784104203","0.733858787339","0.711398006457","0.579602638316","0.734876927298","0.734860210439","0.000033736289","","","","","" -"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","81649","742376","65811","602476","38436668082","65811","38436733893","0.000171","602476","1344852","44.798684","0.918569588474","0.552013158325","0.999998287810","0.999982613933","0.776005723068","0.810879063265","0.689607573295","0.599890587538","0.526260347085","0.712083211201","0.712076031428","0.000017386067","","","","","" -"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","37843","506459","41477","351958","10594922057","41477","10594963534","0.000391","351958","858417","41.000819","0.924303203294","0.589991810507","0.999996085215","0.999962868855","0.794993947861","0.830216831177","0.720244490537","0.635998708058","0.562798507380","0.738465517386","0.738449797528","0.000037131145","","","","","" +"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","81641","742393","65811","602459","38436668082","65811","38436733893","0.000171","602459","1344852","44.797420","0.918571301305","0.552025799121","0.999998287810","0.999982614376","0.776012043466","0.810885586285","0.689617919831","0.599902676509","0.526272398156","0.712092028219","0.712084848565","0.000017385624","","","","","" +"SNOWBALL_GERMAN_DIRECT","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","37842","506464","41477","351953","10594922057","41477","10594963534","0.000391","351953","858417","41.000236","0.924303894032","0.589997635182","0.999996085215","0.999962869327","0.794996860199","0.830219583690","0.720249040429","0.636004188257","0.562804063590","0.738469438547","0.738453718829","0.000037130673","","","","","" "SNOWBALL_GERMAN_LUCENE_FILTER","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","ALL_WORDS","PRIMARY_OUTPUT","54092","277266","1474","277266","0","1","277266","86669","723725","142783","621127","38436591110","142783","38436733893","0.000371","621127","1344852","46.185528","0.835220217240","0.538144717783","0.999996285246","0.999980126218","0.769070501515","0.752174652309","0.654551949931","0.579358576068","0.486493662760","0.670424751999","0.670415952491","0.000019873782","","","","","" "SNOWBALL_GERMAN_LUCENE_FILTER","DE_DE","de-de-default","1.0.0","cbfa038122823f02e4bdb54b0035492c356b6ecd80f11eb11290d7a7248a59f5","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","16007","145574","228","145574","0","1","145574","46077","471644","34482","386773","10594929052","34482","10594963534","0.000325","386773","858417","45.056540","0.931870719939","0.549434598802","0.999996745435","0.999960243292","0.774715672118","0.817996747049","0.691284921032","0.598564290417","0.528216517210","0.715543160924","0.715527026594","0.000039756708","","","","","" "SNOWBALL_HUNGARIAN_DIRECT","HU_HU","hu-hu-default","1.0.0","359d46a01d751ec823705ad7f3dd1cc8f6663feb1a9d13cb04d0c6fb51ab646e","ALL_WORDS","PRIMARY_OUTPUT","19406","910688","1","910688","0","1","910688","116105","14275129","1281527","7842726","414652461946","1281527","414653743473","0.000309","7842726","22117855","35.458800","0.917621949087","0.645411998587","0.999996909404","0.999977996662","0.822704453996","0.846239680964","0.757813631609","0.686119053091","0.610064359819","0.769574048489","0.769564274829","0.000022003338","","","","","" @@ -241,6 +243,10 @@ Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model S "SNOWBALL_NORWEGIAN_NYNORSK_DIRECT","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4681","16906","23","16906","0","1","16906","6120","20847","1201","7458","142868459","1201","142869660","0.000841","7458","28305","26.348702","0.945527939042","0.736512983572","0.999991593737","0.999939404316","0.868252288654","0.894744070663","0.828034079399","0.770581364403","0.706534264217","0.834502009245","0.834474211808","0.000060595684","","","","","" "SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","ALL_WORDS","PRIMARY_OUTPUT","4688","16937","23","16937","0","1","16937","6144","20854","1222","7508","143392932","1222","143394154","0.000852","7508","28362","26.472040","0.944645769161","0.735279599464","0.999991478035","0.999939130896","0.867635538749","0.893747964274","0.826916213966","0.769384020542","0.704908058410","0.833413920441","0.833385994629","0.000060869104","","","","","" "SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER","NN_NO","nn-no-default","1.0.0","900cf2005605aea2a3d8d731ec0b0c1f47fb4469b4ba6b9134145d4d026a0398","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4681","16906","23","16906","0","1","16906","6130","20824","1201","7481","142868459","1201","142869660","0.000841","7481","28305","26.429959","0.945471055619","0.735700406289","0.999991593737","0.999939243362","0.867846000013","0.894463296250","0.827498509835","0.769862102111","0.705754761743","0.834016450529","0.833988591624","0.000060756638","","","","","" +"SNOWBALL_PERSIAN_DIRECT","FA_IR","fa-ir-default","1.0.0","b29a0d168a6a97f980666aa40b74a0edd8b6be4ab3320a7abfbb76b3529f4ea1","ALL_WORDS","PRIMARY_OUTPUT","69","3544","0","3544","0","1","3544","2029","6748","79","89296","6182073","79","6182152","0.001278","89296","96044","92.974054","0.988428299399","0.070259464412","0.999987221278","0.985764222716","0.535123342845","0.273526169012","0.131193436440","0.086290898024","0.070201720712","0.263526930171","0.261598150185","0.014235777284","","","","","" +"SNOWBALL_PERSIAN_DIRECT","FA_IR","fa-ir-default","1.0.0","b29a0d168a6a97f980666aa40b74a0edd8b6be4ab3320a7abfbb76b3529f4ea1","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","69","3544","0","3544","0","1","3544","2029","6748","79","89296","6182073","79","6182152","0.001278","89296","96044","92.974054","0.988428299399","0.070259464412","0.999987221278","0.985764222716","0.535123342845","0.273526169012","0.131193436440","0.086290898024","0.070201720712","0.263526930171","0.261598150185","0.014235777284","","","","","" +"SNOWBALL_POLISH_DIRECT","PL_PL","pl-pl-unimorph","1.0.0","8191ed727097839cc808cbc5c56a1bd78b3c851e7733ad226ad9a51519a54721","ALL_WORDS","PRIMARY_OUTPUT","9990","120867","1","120867","0","1","120867","19265","723037","70656","394036","7303167682","70656","7303238338","0.000967","394036","1117073","35.273970","0.910978174181","0.647260295433","0.999990325388","0.999936381518","0.823625310410","0.842338201869","0.756803292502","0.687038256475","0.608755869394","0.767880200391","0.767851723211","0.000063618482","","","","","" +"SNOWBALL_POLISH_DIRECT","PL_PL","pl-pl-unimorph","1.0.0","8191ed727097839cc808cbc5c56a1bd78b3c851e7733ad226ad9a51519a54721","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","9846","119451","1","119451","0","1","119451","18927","718592","70647","392165","7133029571","70647","7133100218","0.000990","392165","1110757","35.306102","0.910487190826","0.646938979453","0.999990095891","0.999935127795","0.823464537672","0.841893538764","0.756414224030","0.686692785364","0.608252553741","0.767482673444","0.767453627968","0.000064872205","","","","","" "SNOWBALL_PORTUGUESE_DIRECT","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","ALL_WORDS","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","","" "SNOWBALL_PORTUGUESE_DIRECT","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","LOWERCASE_GROUPS_ONLY","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","","" "SNOWBALL_PORTUGUESE_LUCENE_FILTER","PT_PT","pt-pt-default","1.0.0","7a035ff330a6f0548f446cd0d6617bc1cf4751292125a3564d3a255c5d6f516d","ALL_WORDS","PRIMARY_OUTPUT","4001","211091","0","211091","0","1","211091","11315","4816198","146201","670154","22273967042","146201","22274113243","0.000656","670154","5486352","12.214929","0.970538241685","0.877850710272","0.999993436282","0.999963358632","0.938922073277","0.950467296507","0.921870566157","0.894944355740","0.855064834721","0.923031789707","0.923014032222","0.000036641368","","","","","" diff --git a/docs/benchmarks/data/stemming-quality.sha256 b/docs/benchmarks/data/stemming-quality.sha256 index 731eca6..3e853bb 100644 --- a/docs/benchmarks/data/stemming-quality.sha256 +++ b/docs/benchmarks/data/stemming-quality.sha256 @@ -1 +1 @@ -edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8 stemming-quality.csv +d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5 stemming-quality.csv diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index 5b5ed44..f035e62 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -52,8 +52,8 @@ Open [Language Benchmark Pages](languages/index.md) for the complete language li The English dictionary coverage benchmark shows the current contracted-trie operating curve. With the full English dictionary, Radixor reaches `97.478%` all-token exactness and `97.197%` -changed-token exactness at `98.0 ns/token`. Even with a deterministic 10% dictionary slice, it -keeps `92.868%` all-token exactness and `76.516%` changed-token exactness at `80.6 ns/token`. +changed-token exactness at `71.6 ns/token`. Even with a deterministic 10% dictionary slice, it +keeps `92.868%` all-token exactness and `76.516%` changed-token exactness at `47.0 ns/token`. Those figures should not be reduced to a single speed badge. The professional interpretation is a quality/speed envelope: the amount and quality of dictionary knowledge affect stemming precision, @@ -61,7 +61,17 @@ while contracted tries reduce lookup cost in uniform regions of the compiled gra ## Quality versus performance -Each language page keeps exact-root accuracy, JMH latency, and pairwise linguistic-quality results in separate tables. No undocumented scalar combines them. The 2026-07-23 language tables are generated from the unrounded JMH comparison report produced on the environment documented for this refresh. Readers should inspect the quality and speed dimensions side by side; no cross-language Pareto ranking is inferred from workloads with different dictionaries and token counts. +Each language page keeps exact-root accuracy, JMH latency, and pairwise linguistic-quality results in separate tables. No undocumented scalar combines them. The 2026-08-10 language tables are generated exclusively from the current unrounded JMH comparison report produced on the environment documented for this refresh. The Snowball 3.1.0 matrix adds direct Czech, Persian, and Polish stemmers; all previously published Java stemmers were measured again in the same run. Readers should inspect the quality and speed dimensions side by side; no cross-language Pareto ranking is inferred from workloads with different dictionaries and token counts. + +### New Snowball 3.1.0 rows + +| New direct stemmer | All exact | Changed exact | Root preserved | Speed | Relative to same-language Radixor | +| --- | ---: | ---: | ---: | ---: | ---: | +| Czech | 19.865% | 18.186% | 27.645% | 82.4 ns/token | 1.187× | +| Persian | 3.660% | 0.000% | 100.000% | 298.1 ns/token | 6.486× | +| Polish | 22.315% | 20.225% | 34.078% | 86.5 ns/token | 1.196× | + +These rows describe exact agreement with each Radixor model dictionary and the measured direct API workload; they are not a universal linguistic ranking. In this dataset the three new Snowball stemmers are both less exact and slower than their same-language Radixor baseline. Lucene 10.5.0 does not expose the three new algorithms through `SnowballFilter`, so no synthetic Lucene wrapper rows were added. @@ -76,16 +86,16 @@ The validated snapshot is a broad multilingual comparison covering the complete | Language | Dictionary mode | Winner | Balanced accuracy | Runner-up | Difference | Exact tie | Deterministic stemmers | |---|---|---|---:|---|---:|---|---:| -|Czech (`CS_CZ`)|ALL_WORDS|Radixor|0.996617|HUNSPELL CZECH LUCENE FILTER|0.142485045|no|3| -|Czech (`CS_CZ`)|LOWERCASE_GROUPS_ONLY|Radixor|0.997195|HUNSPELL CZECH LUCENE FILTER|0.144045088|no|3| -|Danish (`DA_DK`)|ALL_WORDS|Radixor|0.996243|SNOWBALL DANISH LUCENE FILTER|0.058337376|no|3| -|Danish (`DA_DK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.996482|SNOWBALL DANISH DIRECT|0.058471663|no|3| +|Czech (`CS_CZ`)|ALL_WORDS|Radixor|0.996617|HUNSPELL CZECH LUCENE FILTER|0.142485045|no|4| +|Czech (`CS_CZ`)|LOWERCASE_GROUPS_ONLY|Radixor|0.997195|HUNSPELL CZECH LUCENE FILTER|0.144045088|no|4| +|Danish (`DA_DK`)|ALL_WORDS|Radixor|0.996243|SNOWBALL DANISH DIRECT|0.053760569|no|3| +|Danish (`DA_DK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.996482|SNOWBALL DANISH DIRECT|0.054099342|no|3| |Dutch (`NL_NL`)|ALL_WORDS|Radixor|0.988733|SNOWBALL DUTCH DIRECT|0.261639748|no|4| |Dutch (`NL_NL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.989114|SNOWBALL DUTCH DIRECT|0.258605347|no|4| |English (`US_UK`)|ALL_WORDS|Radixor|0.965537|ENGLISH LUCENE PORTER COPIED|0.010741250|no|11| |English (`US_UK`)|LOWERCASE_GROUPS_ONLY|Radixor|0.966202|ENGLISH LUCENE PORTER COPIED|0.011138557|no|11| |Finnish (`FI_FI`)|ALL_WORDS|Radixor|0.984838|SNOWBALL FINNISH LUCENE FILTER|0.244558928|no|4| -|Finnish (`FI_FI`)|LOWERCASE_GROUPS_ONLY|Radixor|0.988242|SNOWBALL FINNISH DIRECT|0.249897933|no|4| +|Finnish (`FI_FI`)|LOWERCASE_GROUPS_ONLY|Radixor|0.988242|SNOWBALL FINNISH DIRECT|0.249699076|no|4| |French (`FR_FR`)|ALL_WORDS|Radixor|0.958627|SNOWBALL FRENCH DIRECT|0.109964908|no|6| |French (`FR_FR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.958856|SNOWBALL FRENCH DIRECT|0.110030565|no|6| |German (`DE_DE`)|ALL_WORDS|Radixor|0.910445|GERMAN CISTEM|0.031918024|no|8| @@ -100,10 +110,10 @@ The validated snapshot is a broad multilingual comparison covering the complete |Norwegian Bokmal (`NB_NO`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976240|SNOWBALL NORWEGIAN BOKMAL DIRECT|0.101954266|no|5| |Norwegian Nynorsk (`NN_NO`)|ALL_WORDS|Radixor|0.950991|SNOWBALL NORWEGIAN NYNORSK DIRECT|0.082896791|no|3| |Norwegian Nynorsk (`NN_NO`)|LOWERCASE_GROUPS_ONLY|Radixor|0.951104|SNOWBALL NORWEGIAN NYNORSK DIRECT|0.082851757|no|3| -|Persian (`FA_IR`)|ALL_WORDS|Radixor|0.976360|PERSIAN LUCENE PERSIAN STEM FILTER|0.474147508|no|2| -|Persian (`FA_IR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976360|PERSIAN LUCENE PERSIAN STEM FILTER|0.474147508|no|2| -|Polish (`PL_PL`)|ALL_WORDS|Radixor|0.991105|POLISH LUCENE MORFOLOGIK FILTER|0.042712804|no|5| -|Polish (`PL_PL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.991301|POLISH LUCENE MORFOLOGIK FILTER|0.042883749|no|5| +|Persian (`FA_IR`)|ALL_WORDS|Radixor|0.976360|SNOWBALL PERSIAN DIRECT|0.441236451|no|3| +|Persian (`FA_IR`)|LOWERCASE_GROUPS_ONLY|Radixor|0.976360|SNOWBALL PERSIAN DIRECT|0.441236451|no|3| +|Polish (`PL_PL`)|ALL_WORDS|Radixor|0.991105|POLISH LUCENE MORFOLOGIK FILTER|0.042712804|no|6| +|Polish (`PL_PL`)|LOWERCASE_GROUPS_ONLY|Radixor|0.991301|POLISH LUCENE MORFOLOGIK FILTER|0.042883749|no|6| |Portuguese (`PT_PT`)|ALL_WORDS|Radixor|0.998542|SNOWBALL PORTUGUESE DIRECT|0.059619854|no|6| |Portuguese (`PT_PT`)|LOWERCASE_GROUPS_ONLY|Radixor|0.998542|SNOWBALL PORTUGUESE DIRECT|0.059619854|no|6| |Russian (`RU_RU`)|ALL_WORDS|Radixor|0.990188|SNOWBALL RUSSIAN LUCENE FILTER|0.155623602|no|4| @@ -166,7 +176,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |ITALIAN LUCENE ITALIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| |NORWEGIAN BOKMAL LUCENE NORWEGIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| |NORWEGIAN BOKMAL LUCENE NORWEGIAN MINIMAL STEM FILTER|1|0|0|0|5.000|5.000| -|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|2.000|2.000| +|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|3.000|3.000| |POLISH LUCENE MORFOLOGIK FILTER|1|0|0|1|2.000|2.000| |POLISH LUCENE STEMPEL DIRECT|1|0|0|0|4.000|4.000| |POLISH LUCENE STEMPEL FILTER|1|0|0|0|5.000|5.000| @@ -174,8 +184,9 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |PORTUGUESE LUCENE PORTUGUESE MINIMAL STEM FILTER|1|0|0|0|6.000|6.000| |PORTUGUESE LUCENE PORTUGUESE STEM FILTER|1|0|0|0|4.000|4.000| |RUSSIAN LUCENE RUSSIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| -|SNOWBALL DANISH DIRECT|1|0|0|1|3.000|3.000| -|SNOWBALL DANISH LUCENE FILTER|1|0|0|1|2.000|2.000| +|SNOWBALL CZECH DIRECT|1|0|0|0|4.000|4.000| +|SNOWBALL DANISH DIRECT|1|0|0|1|2.000|2.000| +|SNOWBALL DANISH LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL DUTCH DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL DUTCH LUCENE FILTER|1|0|0|0|4.000|4.000| |SNOWBALL FINNISH DIRECT|1|0|0|1|3.000|3.000| @@ -192,6 +203,8 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |SNOWBALL NORWEGIAN BOKMAL LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL NORWEGIAN NYNORSK DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL NORWEGIAN NYNORSK LUCENE FILTER|1|0|0|1|3.000|3.000| +|SNOWBALL PERSIAN DIRECT|1|0|0|1|2.000|2.000| +|SNOWBALL POLISH DIRECT|1|0|0|0|6.000|6.000| |SNOWBALL PORTUGUESE DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL PORTUGUESE LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL RUSSIAN DIRECT|1|0|0|1|3.000|3.000| @@ -246,7 +259,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |ITALIAN LUCENE ITALIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| |NORWEGIAN BOKMAL LUCENE NORWEGIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| |NORWEGIAN BOKMAL LUCENE NORWEGIAN MINIMAL STEM FILTER|1|0|0|0|5.000|5.000| -|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|2.000|2.000| +|PERSIAN LUCENE PERSIAN STEM FILTER|1|0|0|1|3.000|3.000| |POLISH LUCENE MORFOLOGIK FILTER|1|0|0|1|2.000|2.000| |POLISH LUCENE STEMPEL DIRECT|1|0|0|0|4.000|4.000| |POLISH LUCENE STEMPEL FILTER|1|0|0|0|5.000|5.000| @@ -254,6 +267,7 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |PORTUGUESE LUCENE PORTUGUESE MINIMAL STEM FILTER|1|0|0|0|6.000|6.000| |PORTUGUESE LUCENE PORTUGUESE STEM FILTER|1|0|0|0|4.000|4.000| |RUSSIAN LUCENE RUSSIAN LIGHT STEM FILTER|1|0|0|0|4.000|4.000| +|SNOWBALL CZECH DIRECT|1|0|0|0|4.000|4.000| |SNOWBALL DANISH DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL DANISH LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL DUTCH DIRECT|1|0|0|1|2.000|2.000| @@ -272,6 +286,8 @@ Counts use `PRIMARY_OUTPUT` only and retain each adapter configuration as a sepa |SNOWBALL NORWEGIAN BOKMAL LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL NORWEGIAN NYNORSK DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL NORWEGIAN NYNORSK LUCENE FILTER|1|0|0|1|3.000|3.000| +|SNOWBALL PERSIAN DIRECT|1|0|0|1|2.000|2.000| +|SNOWBALL POLISH DIRECT|1|0|0|0|6.000|6.000| |SNOWBALL PORTUGUESE DIRECT|1|0|0|1|2.000|2.000| |SNOWBALL PORTUGUESE LUCENE FILTER|1|0|0|1|3.000|3.000| |SNOWBALL RUSSIAN DIRECT|1|0|0|1|2.000|2.000| @@ -305,7 +321,7 @@ These aggregates cover all 20 documented languages. Macro balanced accuracy give ### Reproducible data - [Machine-readable quality snapshot](data/stemming-quality.csv) -- SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - [Linguistic quality methodology](reference/linguistic-quality.md) - [Tested stemmer inventory](reference/tested-stemmers.md) - [Reproducibility and raw data](reference/reproducibility.md) diff --git a/docs/benchmarks/languages/czech.md b/docs/benchmarks/languages/czech.md index 26be2fd..169d128 100644 --- a/docs/benchmarks/languages/czech.md +++ b/docs/benchmarks/languages/czech.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `cs-cz-default` | `1.0.0` | `CS_CZ` | 5,113 | 56,612 | 10,049 | 46,563 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `cs-cz-default` | `1.0.0` | `CS_CZ` | 5,113 | 56,612 | 10,049 | 46,563 | 46,563 | ## Radixor Patch Command Distribution @@ -30,14 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.465% | 99.439% | 99.582% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 99.465% | 99.439% | 99.582% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 84.850% | 82.269% | 96.806% | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene CzechStemFilter | 16.784% | 15.538% | 22.559% | Lucene Czech suffix stemmer implemented as a TokenFilter. | - - - - - +| Official Snowball direct | 19.865% | 18.186% | 27.645% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. | ## Speed @@ -45,18 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `czechRadixor` | 3.395 | 0.066 | 72.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 381.189 | 32.563 | 8186.5 | 112.265 | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Lucene CzechStemFilter | `czechLuceneCzechStemFilter` | 3.125 | 0.042 | 67.1 | 0.920 | Czech suffix stemmer implemented as a Lucene TokenFilter. | - - - - - +| Radixor | `czechRadixor` | 3.230 | 0.050 | 69.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 349.111 | 24.459 | 7497.6 | 108.091 | Benchmark-only Czech Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Lucene CzechStemFilter | `czechLuceneCzechStemFilter` | 2.927 | 0.032 | 62.9 | 0.906 | Czech suffix stemmer implemented as a Lucene TokenFilter. | +| Official Snowball direct | `snowballDirect[CZECH]` | 3.835 | 0.320 | 82.4 | 1.187 | Official Snowball 3.1.0 generated Java stemmer; direct API. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -74,11 +66,11 @@ Runtime performance and linguistic grouping quality are independent dimensions. The default model is `cs-cz-default`, loaded from classpath resource `org/egothor/stemmer/models/cs-cz-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses. -- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996617** among 3 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.854132, a difference of 0.142485. This rank does not imply leadership in throughput or every secondary metric. -- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.997195** among 3 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.853150, a difference of 0.144045. This rank does not imply leadership in throughput or every secondary metric. +- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996617** among 4 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.854132, a difference of 0.142485. This rank does not imply leadership in throughput or every secondary metric. +- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.997195** among 4 deterministic stemmers. The runner-up is `HUNSPELL CZECH LUCENE FILTER` at 0.853150, a difference of 0.144045. This rank does not imply leadership in throughput or every secondary metric. ### `ALL_WORDS` -This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **8 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -89,6 +81,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|0.996617|0.000000%|0.676519%| |2|HUNSPELL CZECH LUCENE FILTER|0.854132|0.000691%|29.172837%| |3|CZECH LUCENE CZECH STEM FILTER|0.794343|0.000928%|41.130549%| +|4|SNOWBALL CZECH DIRECT|0.786366|0.000904%|42.725842%| @@ -99,6 +92,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|1.000000|0.993235|1.000000|0.996617|0.999998|0.000002| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.958877|0.708272|0.999993|0.854132|0.999927|0.000073| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.935210|0.588695|0.999991|0.794343|0.999897|0.000103| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.935153|0.572742|0.999991|0.786366|0.999894|0.000106| @@ -109,6 +103,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|0.998640|0.996606|0.994581|0.993235|0.996612|0.996611| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.895506|0.814739|0.747335|0.687392|0.824103|0.824070| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.836710|0.722556|0.635811|0.565626|0.741992|0.741949| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.830101|0.710396|0.620864|0.550864|0.731848|0.731804| @@ -119,6 +114,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|298476|0|2033|1320705191|0 / 1320705191|2033 / 300509| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|212842|9128|87667|1320696063|9128 / 1320705191|87667 / 300509| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|176908|12256|123601|1320692935|12256 / 1320705191|123601 / 300509| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|172114|11935|128395|1320693256|11935 / 1320705191|128395 / 300509| @@ -193,7 +189,7 @@ Alternative candidates are capability analyses, not replacements for the determi ### `LOWERCASE_GROUPS_ONLY` -This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **8 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -204,6 +200,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|0.997195|0.000000%|0.561033%| |2|HUNSPELL CZECH LUCENE FILTER|0.853150|0.000700%|29.369351%| |3|CZECH LUCENE CZECH STEM FILTER|0.792522|0.000918%|41.494586%| +|4|SNOWBALL CZECH DIRECT|0.784821|0.000923%|43.034822%| @@ -214,6 +211,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|1.000000|0.994390|1.000000|0.997195|0.999999|0.000001| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.958957|0.706306|0.999993|0.853150|0.999925|0.000075| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.936557|0.585054|0.999991|0.792522|0.999895|0.000105| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.934577|0.569652|0.999991|0.784821|0.999891|0.000109| @@ -224,6 +222,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|0.998873|0.997187|0.995507|0.994390|0.997191|0.997190| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|0.894932|0.813466|0.745594|0.685581|0.822993|0.822960| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|0.836092|0.720206|0.632534|0.562751|0.740227|0.740184| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|0.828436|0.707849|0.617907|0.547807|0.729646|0.729601| @@ -234,6 +233,7 @@ This mode contains **7 result rows**, **3 evaluated stemmers**, and **3 output p |1|Radixor|PRIMARY_OUTPUT|295818|0|1669|1284770069|0 / 1284770069|1669 / 297487| |2|HUNSPELL CZECH LUCENE FILTER|PRIMARY_OUTPUT|210117|8993|87370|1284761076|8993 / 1284770069|87370 / 297487| |3|CZECH LUCENE CZECH STEM FILTER|PRIMARY_OUTPUT|174046|11790|123441|1284758279|11790 / 1284770069|123441 / 297487| +|4|SNOWBALL CZECH DIRECT|PRIMARY_OUTPUT|169464|11863|128023|1284758206|11863 / 1284770069|128023 / 297487| @@ -330,7 +330,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `CS_CZ` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/danish.md b/docs/benchmarks/languages/danish.md index ad74bfe..554d575 100644 --- a/docs/benchmarks/languages/danish.md +++ b/docs/benchmarks/languages/danish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `da-dk-default` | `1.0.0` | `DA_DK` | 4,179 | 32,256 | 8,356 | 23,900 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `da-dk-default` | `1.0.0` | `DA_DK` | 4,179 | 32,256 | 8,356 | 23,900 | 23,900 | ## Radixor Patch Command Distribution @@ -30,14 +30,9 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.371% | 99.527% | 98.923% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 99.371% | 99.527% | 98.923% | Radixor dictionary-trained patch-command stemmer. | | Lucene SnowballFilter | 55.509% | 54.159% | 59.371% | Lucene TokenFilter integration path around the Snowball algorithm. | -| Official Snowball direct | 55.509% | 54.159% | 59.371% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | - - - - - +| Official Snowball direct | 55.971% | 54.791% | 59.347% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | ## Speed @@ -45,18 +40,13 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `radixor[DANISH]` | 1.206 | 0.134 | 50.5 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Official Snowball direct | `snowballDirect[DANISH]` | 2.326 | 0.205 | 97.3 | 1.928 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[DANISH]` | 3.275 | 0.335 | 137.0 | 2.716 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - - - +| Radixor | `radixor[DANISH]` | 1.146 | 0.122 | 47.9 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Official Snowball direct | `snowballDirect[DANISH]` | 2.542 | 0.179 | 106.4 | 2.219 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[DANISH]` | 2.879 | 0.239 | 120.4 | 2.512 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -74,8 +64,8 @@ Runtime performance and linguistic grouping quality are independent dimensions. The default model is `da-dk-default`, loaded from classpath resource `org/egothor/stemmer/models/da-dk-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses. -- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996243** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH LUCENE FILTER` at 0.937905, a difference of 0.058337. This rank does not imply leadership in throughput or every secondary metric. -- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.996482** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.938010, a difference of 0.058472. This rank does not imply leadership in throughput or every secondary metric. +- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.996243** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.942482, a difference of 0.053761. This rank does not imply leadership in throughput or every secondary metric. +- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.996482** among 3 deterministic stemmers. The runner-up is `SNOWBALL DANISH DIRECT` at 0.942383, a difference of 0.054099. This rank does not imply leadership in throughput or every secondary metric. ### `ALL_WORDS` This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. @@ -87,8 +77,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) | |---:|---|---:|---:|---:| |1|Radixor|0.996243|0.000000%|0.751435%| -|2|SNOWBALL DANISH LUCENE FILTER|0.937905|0.001273%|12.417638%| -|3|SNOWBALL DANISH DIRECT|0.937839|0.001230%|12.431016%| +|2|SNOWBALL DANISH DIRECT|0.942482|0.001236%|11.502313%| +|3|SNOWBALL DANISH LUCENE FILTER|0.937905|0.001273%|12.417638%| @@ -97,8 +87,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|1.000000|0.992486|1.000000|0.996243|0.999998|0.000002| -|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.940600|0.875824|0.999987|0.937905|0.999959|0.000041| -|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942465|0.875690|0.999988|0.937839|0.999959|0.000041| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942799|0.884977|0.999988|0.942482|0.999961|0.000039| +|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.940600|0.875824|0.999987|0.937905|0.999959|0.000041| @@ -107,8 +97,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.998488|0.996229|0.993979|0.992486|0.996236|0.996235| -|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.926889|0.907057|0.888055|0.829921|0.907634|0.907614| -|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.928307|0.907851|0.888277|0.831252|0.908464|0.908444| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.930638|0.912973|0.895967|0.839881|0.913430|0.913411| +|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.926889|0.907057|0.888055|0.829921|0.907634|0.907614| @@ -117,8 +107,8 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|89021|0|674|389687465|0 / 389687465|674 / 89695| -|2|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78557|4961|11138|389682504|4961 / 389687465|11138 / 89695| -|3|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|78545|4795|11150|389682670|4795 / 389687465|11150 / 89695| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|79378|4816|10317|389682649|4816 / 389687465|10317 / 89695| +|3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78557|4961|11138|389682504|4961 / 389687465|11138 / 89695| @@ -195,7 +185,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) | |---:|---|---:|---:|---:| |1|Radixor|0.996482|0.000000%|0.703596%| -|2|SNOWBALL DANISH DIRECT|0.938010|0.001235%|12.396694%| +|2|SNOWBALL DANISH DIRECT|0.942383|0.001240%|11.522225%| |3|SNOWBALL DANISH LUCENE FILTER|0.938010|0.001235%|12.396694%| @@ -205,7 +195,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|1.000000|0.992964|1.000000|0.996482|0.999998|0.000002| -|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942392|0.876033|0.999988|0.938010|0.999959|0.000041| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.942693|0.884778|0.999988|0.942383|0.999961|0.000039| |3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.942392|0.876033|0.999988|0.938010|0.999959|0.000041| @@ -215,7 +205,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.998585|0.996470|0.994363|0.992964|0.996476|0.996475| -|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.928328|0.908002|0.888547|0.831505|0.908607|0.908587| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|0.930511|0.912818|0.895784|0.839618|0.913277|0.913257| |3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|0.928328|0.908002|0.888547|0.831505|0.908607|0.908587| @@ -225,7 +215,7 @@ This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|88910|0|630|388404335|0 / 388404335|630 / 89540| -|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|78440|4795|11100|388399540|4795 / 388404335|11100 / 89540| +|2|SNOWBALL DANISH DIRECT|PRIMARY_OUTPUT|79223|4816|10317|388399519|4816 / 388404335|10317 / 89540| |3|SNOWBALL DANISH LUCENE FILTER|PRIMARY_OUTPUT|78440|4795|11100|388399540|4795 / 388404335|11100 / 89540| @@ -316,7 +306,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `DA_DK` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/dutch.md b/docs/benchmarks/languages/dutch.md index fc6a41a..658f1c7 100644 --- a/docs/benchmarks/languages/dutch.md +++ b/docs/benchmarks/languages/dutch.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `nl-nl-default` | `1.0.0` | `NL_NL` | 4,992 | 31,466 | 9,981 | 21,485 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `nl-nl-default` | `1.0.0` | `NL_NL` | 4,992 | 31,466 | 9,981 | 21,485 | 21,485 | ## Radixor Patch Command Distribution @@ -30,35 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.120% | 98.711% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 99.120% | 98.711% | 100.000% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 46.590% | 22.718% | 97.976% | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. | | Official Snowball direct | 15.954% | 8.992% | 30.939% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene SnowballFilter | 12.620% | 5.441% | 28.073% | Lucene TokenFilter integration path around the Snowball algorithm. | - - - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `radixor[DUTCH]` | 1.410 | 0.139 | 65.6 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 24.183 | 2.889 | 1125.6 | 17.156 | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Official Snowball direct | `snowballDirect[DUTCH]` | 4.560 | 0.205 | 212.2 | 3.235 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[DUTCH]` | 7.762 | 0.262 | 361.3 | 5.506 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - - - +| Radixor | `radixor[DUTCH]` | 1.340 | 0.127 | 62.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 22.275 | 2.325 | 1036.8 | 16.621 | Benchmark-only Dutch Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Official Snowball direct | `snowballDirect[DUTCH]` | 4.298 | 0.185 | 200.0 | 3.207 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[DUTCH]` | 7.317 | 0.255 | 340.6 | 5.460 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -340,7 +330,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `NL_NL` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/english.md b/docs/benchmarks/languages/english.md index 1e35a3f..ec0067d 100644 --- a/docs/benchmarks/languages/english.md +++ b/docs/benchmarks/languages/english.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `us-uk-default` | `1.0.0` | `US_UK` | 396,939 | 1,004,374 | 793,874 | 210,500 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `us-uk-default` | `1.0.0` | `US_UK` | 396,939 | 1,004,374 | 793,874 | 210,500 | 210,500 | ## Radixor Patch Command Distribution @@ -30,47 +30,39 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 97.478% | 97.197% | 97.552% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 97.478% | 97.197% | 97.552% | Radixor dictionary-trained patch-command stemmer. | | Lucene EnglishMinimalStemFilter | 90.981% | 65.189% | 97.820% | Minimal English plural reduction, not a full stemmer. | | Lucene KStemFilter | 80.076% | 76.608% | 80.996% | Krovetz-style English stemming TokenFilter; broader than minimal suffix reducers. | | Lucene HunspellStemFilter | 80.243% | 12.750% | 98.139% | Benchmark-only English Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene EnglishPossessiveFilter | 79.032% | 0.003% | 99.987% | Possessive-ending remover only, not a full stemmer. | -| Snowball English / Porter2 | 40.342% | 46.296% | 38.763% | Porter2 rule-based suffix stemmer, distinct from original Porter. | +| Snowball English / Porter2 | 40.346% | 46.302% | 38.767% | Porter2 rule-based suffix stemmer, distinct from original Porter. | | Lucene PorterStemFilter | 39.538% | 46.201% | 37.772% | Lucene TokenFilter path for Porter suffix rules; not dictionary-root equivalent. | | Lucene PorterStemmer direct copy | 39.538% | 46.201% | 37.772% | Direct Porter suffix-rule implementation generated under build for benchmark-only use. | | OpenNLP PorterStemmer | 39.538% | 46.201% | 37.772% | Apache OpenNLP Porter suffix-rule implementation. | | Snowball original Porter | 39.529% | 46.179% | 37.766% | Classic Porter rule-based suffix stemmer. | | Paice/Husk Lancaster | 28.055% | 37.039% | 25.673% | Aggressive Paice/Husk rule stemmer that often produces shorter stems. | - - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `radixorUsUkProfiPreferredStem` | 17.489 | 1.380 | 83.1 | 1.000 | Full dictionary patch-command stemmer using compiled patch commands. | -| Lucene EnglishPossessiveFilter | `luceneEnglishPossessiveFilter` | 17.151 | 0.215 | 81.5 | 0.981 | Possessive-ending remover only; not a full stemmer. | -| Lucene EnglishMinimalStemFilter | `luceneEnglishMinimalStemFilter` | 18.522 | 0.152 | 88.0 | 1.059 | Narrow plural reduction filter; not a full stemmer. | -| Lucene PorterStemmer direct copy | `lucenePorterStemmerCopied` | 17.651 | 0.129 | 83.9 | 1.009 | Benchmark-only generated copy of Lucene package-private Porter implementation. | -| OpenNLP PorterStemmer | `opennlpPorterStemmer` | 17.681 | 0.139 | 84.0 | 1.011 | Apache OpenNLP Porter implementation. | -| Snowball original Porter | `snowballOriginalPorter` | 33.290 | 1.916 | 158.1 | 1.904 | Classic Porter suffix-rule stemmer; historical English baseline, not a dictionary-equivalent stemmer. | -| Lucene PorterStemFilter | `lucenePorterStemFilter` | 32.408 | 0.412 | 154.0 | 1.853 | Lucene TokenFilter integration path for Porter; includes TokenStream overhead. | -| Lucene KStemFilter | `luceneKStemFilter` | 45.877 | 0.425 | 217.9 | 2.623 | Krovetz-style English TokenFilter; broader than minimal suffix filters. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 76.852 | 1.028 | 365.1 | 4.394 | Benchmark-only English Hunspell comparison using the benchmark Hunspell corpus. | -| Snowball English / Porter2 | `snowballEnglishPorter2` | 46.568 | 2.414 | 221.2 | 2.663 | Porter2 suffix-rule stemmer, distinct from original Porter. | -| Paice/Husk Lancaster | `paiceHuskLancaster` | 144.951 | 2.710 | 688.6 | 8.288 | Aggressive rule-based English stemmer. | - - - - +| Radixor | `radixorUsUkProfiPreferredStem` | 14.397 | 0.915 | 68.4 | 1.000 | Full dictionary patch-command stemmer using compiled patch commands. | +| Lucene EnglishPossessiveFilter | `luceneEnglishPossessiveFilter` | 15.034 | 0.322 | 71.4 | 1.044 | Possessive-ending remover only; not a full stemmer. | +| Lucene EnglishMinimalStemFilter | `luceneEnglishMinimalStemFilter` | 16.352 | 0.244 | 77.7 | 1.136 | Narrow plural reduction filter; not a full stemmer. | +| Lucene PorterStemmer direct copy | `lucenePorterStemmerCopied` | 16.491 | 0.149 | 78.3 | 1.145 | Benchmark-only generated copy of Lucene package-private Porter implementation. | +| OpenNLP PorterStemmer | `opennlpPorterStemmer` | 16.481 | 0.175 | 78.3 | 1.145 | Apache OpenNLP Porter implementation. | +| Snowball original Porter | `snowballOriginalPorter` | 30.634 | 1.620 | 145.5 | 2.128 | Classic Porter suffix-rule stemmer; historical English baseline, not a dictionary-equivalent stemmer. | +| Lucene PorterStemFilter | `lucenePorterStemFilter` | 29.666 | 0.536 | 140.9 | 2.061 | Lucene TokenFilter integration path for Porter; includes TokenStream overhead. | +| Lucene KStemFilter | `luceneKStemFilter` | 41.485 | 0.509 | 197.1 | 2.882 | Krovetz-style English TokenFilter; broader than minimal suffix filters. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 74.399 | 1.223 | 353.4 | 5.168 | Benchmark-only English Hunspell comparison using the benchmark Hunspell corpus. | +| Snowball English / Porter2 | `snowballEnglishPorter2` | 43.117 | 1.983 | 204.8 | 2.995 | Porter2 suffix-rule stemmer, distinct from original Porter. | +| Paice/Husk Lancaster | `paiceHuskLancaster` | 137.952 | 2.443 | 655.4 | 9.582 | Aggressive rule-based English stemmer. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -104,7 +96,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|0.954796|0.000207%|9.040545%| |3|ENGLISH LUCENE PORTER FILTER|0.954796|0.000207%|9.040545%| |4|ENGLISH OPENNLP PORTER|0.954796|0.000207%|9.040545%| -|5|ENGLISH SNOWBALL PORTER2|0.954708|0.000212%|9.058097%| +|5|ENGLISH SNOWBALL PORTER2|0.954732|0.000212%|9.053310%| |6|ENGLISH SNOWBALL ORIGINAL PORTER|0.954659|0.000206%|9.067990%| |7|ENGLISH PAICE HUSK LANCASTER|0.952535|0.000960%|9.492110%| |8|ENGLISH LUCENE KSTEM FILTER|0.878645|0.000110%|24.270875%| @@ -122,7 +114,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.440121|0.909595|0.999998|0.954796|0.999998|0.000002| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.434174|0.909419|0.999998|0.954708|0.999998|0.000002| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.434309|0.909467|0.999998|0.954732|0.999998|0.000002| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.441440|0.909320|0.999998|0.954659|0.999998|0.000002| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.144284|0.905079|0.999990|0.952535|0.999990|0.000010| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.551014|0.757291|0.999999|0.878645|0.999998|0.000002| @@ -140,7 +132,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.490783|0.593208|0.749662|0.421675|0.632717|0.632716| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.484849|0.587747|0.746086|0.416176|0.628368|0.628367| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.484986|0.587880|0.746192|0.416310|0.628482|0.628481| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.492079|0.594348|0.750277|0.422827|0.633570|0.633569| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.173443|0.248891|0.440518|0.142133|0.361370|0.361368| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.582762|0.637891|0.704541|0.468312|0.645971|0.645970| @@ -158,7 +150,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|285026|362583|28329|175199061547|362583 / 175199424130|28329 / 313355| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|284971|371381|28384|175199052749|371381 / 175199424130|28384 / 313355| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|284986|371197|28369|175199052933|371197 / 175199424130|28369 / 313355| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|284940|360538|28415|175199063592|360538 / 175199424130|28415 / 313355| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|283611|1682034|29744|175197742096|1682034 / 175199424130|29744 / 313355| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|237301|193361|76054|175199230769|193361 / 175199424130|76054 / 313355| @@ -251,7 +243,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|0.955064|0.000222%|8.987032%| |3|ENGLISH LUCENE PORTER FILTER|0.955064|0.000222%|8.987032%| |4|ENGLISH OPENNLP PORTER|0.955064|0.000222%|8.987032%| -|5|ENGLISH SNOWBALL PORTER2|0.955016|0.000228%|8.996666%| +|5|ENGLISH SNOWBALL PORTER2|0.955040|0.000228%|8.991849%| |6|ENGLISH SNOWBALL ORIGINAL PORTER|0.954926|0.000221%|9.014651%| |7|ENGLISH PAICE HUSK LANCASTER|0.952850|0.001032%|9.428933%| |8|ENGLISH LUCENE KSTEM FILTER|0.881028|0.000120%|23.794246%| @@ -269,7 +261,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.440920|0.910130|0.999998|0.955064|0.999998|0.000002| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.435017|0.910033|0.999998|0.955016|0.999998|0.000002| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.435153|0.910082|0.999998|0.955040|0.999998|0.000002| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.442235|0.909853|0.999998|0.954926|0.999998|0.000002| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.144700|0.905711|0.999990|0.952850|0.999990|0.000010| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.551013|0.762058|0.999999|0.881028|0.999998|0.000002| @@ -287,7 +279,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|0.491609|0.594049|0.750417|0.422524|0.633478|0.633477| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.485725|0.588647|0.746915|0.417080|0.629190|0.629189| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|0.485863|0.588782|0.747021|0.417215|0.629305|0.629304| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|0.492900|0.595181|0.751027|0.423671|0.634326|0.634325| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|0.173928|0.249533|0.441413|0.142553|0.362017|0.362015| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|0.583322|0.639575|0.707836|0.470129|0.648000|0.647999| @@ -305,7 +297,7 @@ This mode contains **15 result rows**, **11 evaluated stemmers**, and **3 output |2|ENGLISH LUCENE PORTER COPIED|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382| |3|ENGLISH LUCENE PORTER FILTER|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382| |4|ENGLISH OPENNLP PORTER|PRIMARY_OUTPUT|283398|359344|27984|161561630294|359344 / 161561989638|27984 / 311382| -|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|283368|368027|28014|161561621611|368027 / 161561989638|28014 / 311382| +|5|ENGLISH SNOWBALL PORTER2|PRIMARY_OUTPUT|283383|367843|27999|161561621795|367843 / 161561989638|27999 / 311382| |6|ENGLISH SNOWBALL ORIGINAL PORTER|PRIMARY_OUTPUT|283312|357325|28070|161561632313|357325 / 161561989638|28070 / 311382| |7|ENGLISH PAICE HUSK LANCASTER|PRIMARY_OUTPUT|282022|1666990|29360|161560322648|1666990 / 161561989638|29360 / 311382| |8|ENGLISH LUCENE KSTEM FILTER|PRIMARY_OUTPUT|237291|193354|74091|161561796284|193354 / 161561989638|74091 / 311382| @@ -408,7 +400,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `US_UK` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/finnish.md b/docs/benchmarks/languages/finnish.md index d4c7401..ffca962 100644 --- a/docs/benchmarks/languages/finnish.md +++ b/docs/benchmarks/languages/finnish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `fi-fi-default` | `1.0.0` | `FI_FI` | 57,027 | 1,865,215 | 110,525 | 1,754,690 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `fi-fi-default` | `1.0.0` | `FI_FI` | 57,027 | 1,865,215 | 110,525 | 1,754,690 | 1,754,690 | ## Radixor Patch Command Distribution @@ -30,14 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 98.661% | 98.803% | 96.408% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 98.661% | 98.803% | 96.408% | Radixor dictionary-trained patch-command stemmer. | | Lucene SnowballFilter | 10.991% | 10.268% | 22.471% | Lucene TokenFilter integration path around the Snowball algorithm. | -| Official Snowball direct | 10.991% | 10.268% | 22.471% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | -| Lucene FinnishLightStemFilter | 4.351% | 4.294% | 5.264% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | - - - - +| Official Snowball direct | 10.995% | 10.272% | 22.462% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene FinnishLightStemFilter | 4.351% | 4.294% | 5.264% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | ## Speed @@ -45,18 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `finnishRadixor` | 289.539 | 4.136 | 165.0 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene FinnishLightStemFilter | `finnishLuceneFinnishLightStemFilter` | 175.789 | 4.827 | 100.2 | 0.607 | Light Finnish suffix stemmer. | -| Official Snowball direct | `snowballDirect[FINNISH]` | 259.889 | 8.924 | 148.1 | 0.898 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[FINNISH]` | 332.524 | 9.490 | 189.5 | 1.148 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - - +| Radixor | `finnishRadixor` | 225.954 | 2.940 | 128.8 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene FinnishLightStemFilter | `finnishLuceneFinnishLightStemFilter` | 168.756 | 6.027 | 96.2 | 0.747 | Light Finnish suffix stemmer. | +| Official Snowball direct | `snowballDirect[FINNISH]` | 247.984 | 16.083 | 141.3 | 1.097 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[FINNISH]` | 321.331 | 9.790 | 183.1 | 1.422 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -75,7 +67,7 @@ Runtime performance and linguistic grouping quality are independent dimensions. The default model is `fi-fi-default`, loaded from classpath resource `org/egothor/stemmer/models/fi-fi-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses. - **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.984838** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH LUCENE FILTER` at 0.740279, a difference of 0.244559. This rank does not imply leadership in throughput or every secondary metric. -- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.988242** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH DIRECT` at 0.738344, a difference of 0.249898. This rank does not imply leadership in throughput or every secondary metric. +- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.988242** among 4 deterministic stemmers. The runner-up is `SNOWBALL FINNISH DIRECT` at 0.738543, a difference of 0.249699. This rank does not imply leadership in throughput or every secondary metric. ### `ALL_WORDS` This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. @@ -88,7 +80,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p |---:|---|---:|---:|---:| |1|Radixor|0.984838|<0.000001%|3.032474%| |2|SNOWBALL FINNISH LUCENE FILTER|0.740279|0.000081%|51.944179%| -|3|SNOWBALL FINNISH DIRECT|0.739671|0.000060%|52.065724%| +|3|SNOWBALL FINNISH DIRECT|0.739870|0.000060%|52.025976%| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|0.695725|0.000094%|60.854936%| @@ -99,7 +91,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.999974|0.969675|1.000000|0.984838|0.999999|0.000001| |2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.921471|0.480558|0.999999|0.740279|0.999989|0.000011| -|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.940611|0.479343|0.999999|0.739671|0.999989|0.000011| +|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.940647|0.479740|0.999999|0.739870|0.999989|0.000011| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.890914|0.391451|0.999999|0.695725|0.999987|0.000013| @@ -110,7 +102,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.993763|0.984591|0.975587|0.969650|0.984708|0.984708| |2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.778598|0.631685|0.531413|0.461652|0.665448|0.665443| -|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.788800|0.635056|0.531468|0.465262|0.671472|0.671468| +|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.789035|0.635413|0.531862|0.465645|0.671764|0.671760| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.709787|0.543915|0.440884|0.373546|0.590550|0.590545| @@ -121,7 +113,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|30511413|804|954186|1599841738533|804 / 1599841739337|954186 / 31465599| |2|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|15121052|1288634|16344547|1599840450703|1288634 / 1599841739337|16344547 / 31465599| -|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|15082807|952306|16382792|1599840787031|952306 / 1599841739337|16382792 / 31465599| +|3|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|15095314|952479|16370285|1599840786858|952479 / 1599841739337|16370285 / 31465599| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|12317229|1508153|19148370|1599840231184|1508153 / 1599841739337|19148370 / 31465599| @@ -199,7 +191,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p | Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) | |---:|---|---:|---:|---:| |1|Radixor|0.988242|<0.000001%|2.351587%| -|2|SNOWBALL FINNISH DIRECT|0.738344|0.000062%|52.331112%| +|2|SNOWBALL FINNISH DIRECT|0.738543|0.000062%|52.291340%| |3|SNOWBALL FINNISH LUCENE FILTER|0.738344|0.000062%|52.331112%| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|0.694308|0.000077%|61.138333%| @@ -210,7 +202,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.999973|0.976484|1.000000|0.988242|1.000000|0.000000| -|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.939951|0.476689|0.999999|0.738344|0.999989|0.000011| +|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.939988|0.477087|0.999999|0.738543|0.999989|0.000011| |3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.939951|0.476689|0.999999|0.738344|0.999989|0.000011| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.911893|0.388617|0.999999|0.694308|0.999987|0.000013| @@ -221,7 +213,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.995185|0.988089|0.981093|0.976459|0.988159|0.988159| -|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.786987|0.632573|0.528815|0.462601|0.669376|0.669372| +|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|0.787224|0.632932|0.529209|0.462985|0.669668|0.669664| |3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|0.786987|0.632573|0.528815|0.462601|0.669376|0.669372| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|0.718421|0.544981|0.438999|0.374553|0.595296|0.595291| @@ -232,7 +224,7 @@ This mode contains **6 result rows**, **4 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|30037514|804|723369|1504706134249|804 / 1504706135053|723369 / 30760883| -|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|14663371|936765|16097512|1504705198288|936765 / 1504706135053|16097512 / 30760883| +|2|SNOWBALL FINNISH DIRECT|PRIMARY_OUTPUT|14675605|936938|16085278|1504705198115|936938 / 1504706135053|16085278 / 30760883| |3|SNOWBALL FINNISH LUCENE FILTER|PRIMARY_OUTPUT|14663371|936765|16097512|1504705198288|936765 / 1504706135053|16097512 / 30760883| |4|FINNISH LUCENE FINNISH LIGHT STEM FILTER|PRIMARY_OUTPUT|11954192|1155011|18806691|1504704980042|1155011 / 1504706135053|18806691 / 30760883| @@ -324,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `FI_FI` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/french.md b/docs/benchmarks/languages/french.md index 7056628..24ea717 100644 --- a/docs/benchmarks/languages/french.md +++ b/docs/benchmarks/languages/french.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `fr-fr-default` | `1.0.0` | `FR_FR` | 59,240 | 474,110 | 108,141 | 365,969 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `fr-fr-default` | `1.0.0` | `FR_FR` | 59,240 | 474,110 | 108,141 | 365,969 | 365,969 | ## Radixor Patch Command Distribution @@ -30,16 +30,12 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 94.831% | 94.859% | 94.734% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 94.831% | 94.859% | 94.734% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 68.923% | 63.617% | 86.876% | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene FrenchMinimalStemFilter | 11.472% | 6.236% | 29.192% | Minimal suffix reducer; narrow baseline, not a full stemmer. | | Lucene SnowballFilter | 8.551% | 5.183% | 19.952% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 8.462% | 5.067% | 19.952% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | -| Lucene FrenchLightStemFilter | 6.377% | 3.965% | 14.540% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | - - - - +| Lucene FrenchLightStemFilter | 6.377% | 3.965% | 14.540% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | ## Speed @@ -47,20 +43,16 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `frenchRadixor` | 49.340 | 0.986 | 134.8 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1781.070 | 43.544 | 4866.7 | 36.098 | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Lucene FrenchMinimalStemFilter | `frenchLuceneFrenchMinimalStemFilter` | 19.093 | 0.681 | 52.2 | 0.387 | Minimal French suffix reducer; narrow baseline. | -| Lucene FrenchLightStemFilter | `frenchLuceneFrenchLightStemFilter` | 29.553 | 0.465 | 80.8 | 0.599 | Light French suffix stemmer. | -| Official Snowball direct | `snowballDirect[FRENCH]` | 121.376 | 0.865 | 331.7 | 2.460 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[FRENCH]` | 126.574 | 4.671 | 345.9 | 2.565 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - - +| Radixor | `frenchRadixor` | 37.443 | 0.520 | 102.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1673.192 | 57.385 | 4572.0 | 44.686 | Benchmark-only French Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Lucene FrenchMinimalStemFilter | `frenchLuceneFrenchMinimalStemFilter` | 18.034 | 0.181 | 49.3 | 0.482 | Minimal French suffix reducer; narrow baseline. | +| Lucene FrenchLightStemFilter | `frenchLuceneFrenchLightStemFilter` | 27.961 | 0.493 | 76.4 | 0.747 | Light French suffix stemmer. | +| Official Snowball direct | `snowballDirect[FRENCH]` | 112.255 | 4.045 | 306.7 | 2.998 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[FRENCH]` | 119.555 | 4.560 | 326.7 | 3.193 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -358,7 +350,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `FR_FR` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/german.md b/docs/benchmarks/languages/german.md index be10455..cf2805f 100644 --- a/docs/benchmarks/languages/german.md +++ b/docs/benchmarks/languages/german.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `de-de-default` | `1.0.0` | `DE_DE` | 54,092 | 333,036 | 90,535 | 242,501 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `de-de-default` | `1.0.0` | `DE_DE` | 54,092 | 333,036 | 90,535 | 242,501 | 242,501 | ## Radixor Patch Command Distribution @@ -30,41 +30,33 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 92.725% | 92.847% | 92.396% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 92.725% | 92.847% | 92.396% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 47.064% | 29.661% | 93.678% | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. | | CISTEM (German) | 24.675% | 23.724% | 27.222% | Benchmark-only CISTEM implementation. | -| Lucene GermanLightStemFilter | 37.434% | 35.465% | 42.707% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene GermanLightStemFilter | 37.434% | 35.465% | 42.707% | Light suffix stemmer; intentionally narrower than Radixor's lexicon-trained transformation model. | | Lucene GermanMinimalStemFilter | 27.640% | 24.951% | 34.844% | Minimal suffix reducer; narrow baseline, not a full stemmer. | | Lucene SnowballFilter | 30.956% | 28.853% | 36.589% | Lucene TokenFilter integration path around the Snowball algorithm. | -| Official Snowball direct | 30.481% | 29.027% | 34.376% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Official Snowball direct | 30.483% | 29.030% | 34.376% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene GermanStemFilter | 21.559% | 19.312% | 27.576% | German Lucene stemming TokenFilter; broader than minimal/light variants. | - - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `germanRadixor` | 40.571 | 1.647 | 167.3 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| CISTEM | `germanCistem` | 305.166 | 4.590 | 1258.4 | 7.522 | Benchmark-only CISTEM implementation. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 291.791 | 21.769 | 1203.3 | 7.192 | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Lucene GermanMinimalStemFilter | `germanLuceneGermanMinimalStemFilter` | 23.903 | 0.208 | 98.6 | 0.589 | Minimal German suffix reduction; narrow baseline. | -| Lucene GermanLightStemFilter | `germanLuceneGermanLightStemFilter` | 24.695 | 0.322 | 101.8 | 0.609 | Light German suffix stemmer; narrower than a dictionary stemmer. | -| Lucene GermanStemFilter | `germanLuceneGermanStemFilter` | 72.140 | 1.544 | 297.5 | 1.778 | Older German stemming TokenFilter with normalization requirements. | -| Lucene SnowballFilter | `luceneSnowballFilter[GERMAN]` | 110.086 | 2.315 | 454.0 | 2.713 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | -| Official Snowball direct | `snowballDirect[GERMAN]` | 100.122 | 2.623 | 412.9 | 2.468 | Official Snowball generated Java stemmer; direct API. | - - - - +| Radixor | `germanRadixor` | 27.697 | 0.583 | 114.2 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| CISTEM | `germanCistem` | 289.568 | 8.761 | 1194.1 | 10.455 | Benchmark-only CISTEM implementation. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 265.653 | 10.779 | 1095.5 | 9.591 | Benchmark-only German Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Lucene GermanMinimalStemFilter | `germanLuceneGermanMinimalStemFilter` | 22.385 | 0.217 | 92.3 | 0.808 | Minimal German suffix reduction; narrow baseline. | +| Lucene GermanLightStemFilter | `germanLuceneGermanLightStemFilter` | 23.170 | 0.383 | 95.5 | 0.837 | Light German suffix stemmer; narrower than Radixor's lexicon-trained transformation model. | +| Lucene GermanStemFilter | `germanLuceneGermanStemFilter` | 67.453 | 0.967 | 278.2 | 2.435 | Older German stemming TokenFilter with normalization requirements. | +| Lucene SnowballFilter | `luceneSnowballFilter[GERMAN]` | 105.203 | 2.035 | 433.8 | 3.798 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | +| Official Snowball direct | `snowballDirect[GERMAN]` | 91.847 | 2.301 | 378.7 | 3.316 | Official Snowball generated Java stemmer; direct API. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -96,7 +88,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---:|---:|---:| |1|Radixor|0.910445|0.000002%|17.910967%| |2|GERMAN CISTEM|0.878527|0.000674%|24.293900%| -|3|SNOWBALL GERMAN DIRECT|0.776006|0.000171%|44.798684%| +|3|SNOWBALL GERMAN DIRECT|0.776012|0.000171%|44.797420%| |4|SNOWBALL GERMAN LUCENE FILTER|0.769071|0.000371%|46.185528%| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|0.753833|0.000191%|49.233299%| |6|GERMAN LUCENE GERMAN STEM FILTER|0.720992|0.000443%|55.801084%| @@ -111,7 +103,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.999400|0.820890|1.000000|0.910445|0.999994|0.000006| |2|GERMAN CISTEM|PRIMARY_OUTPUT|0.797231|0.757061|0.999993|0.878527|0.999985|0.000015| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.918570|0.552013|0.999998|0.776006|0.999983|0.000017| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.918571|0.552026|0.999998|0.776012|0.999983|0.000017| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.835220|0.538145|0.999996|0.769071|0.999980|0.000020| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.902792|0.507667|0.999998|0.753833|0.999981|0.000019| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.777304|0.441989|0.999996|0.720992|0.999976|0.000024| @@ -126,7 +118,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.957746|0.901392|0.851302|0.820486|0.905758|0.905755| |2|GERMAN CISTEM|PRIMARY_OUTPUT|0.788860|0.776627|0.764768|0.634824|0.776886|0.776879| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.810879|0.689608|0.599891|0.526260|0.712083|0.712076| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.810886|0.689618|0.599903|0.526272|0.712092|0.712085| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.752175|0.654552|0.579359|0.486494|0.670425|0.670416| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.781189|0.649884|0.556368|0.481355|0.676991|0.676984| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.674901|0.563540|0.483723|0.392311|0.586140|0.586130| @@ -141,7 +133,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|1103976|663|240876|38436733230|663 / 38436733893|240876 / 1344852| |2|GERMAN CISTEM|PRIMARY_OUTPUT|1018135|258954|326717|38436474939|258954 / 38436733893|326717 / 1344852| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|742376|65811|602476|38436668082|65811 / 38436733893|602476 / 1344852| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|742393|65811|602459|38436668082|65811 / 38436733893|602459 / 1344852| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|723725|142783|621127|38436591110|142783 / 38436733893|621127 / 1344852| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|682737|73514|662115|38436660379|73514 / 38436733893|662115 / 1344852| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|594410|170297|750442|38436563596|170297 / 38436733893|750442 / 1344852| @@ -231,7 +223,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---:|---:|---:| |1|Radixor|0.966959|0.000001%|6.608210%| |2|GERMAN CISTEM|0.914727|0.000812%|17.053716%| -|3|SNOWBALL GERMAN DIRECT|0.794994|0.000391%|41.000819%| +|3|SNOWBALL GERMAN DIRECT|0.794997|0.000391%|41.000236%| |4|SNOWBALL GERMAN LUCENE FILTER|0.774716|0.000325%|45.056540%| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|0.768968|0.000130%|46.206331%| |6|GERMAN LUCENE GERMAN STEM FILTER|0.716147|0.000358%|56.770194%| @@ -246,7 +238,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.999900|0.933918|1.000000|0.966959|0.999995|0.000005| |2|GERMAN CISTEM|PRIMARY_OUTPUT|0.892172|0.829463|0.999992|0.914727|0.999978|0.000022| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.924303|0.589992|0.999996|0.794994|0.999963|0.000037| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.924304|0.589998|0.999996|0.794997|0.999963|0.000037| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.931871|0.549435|0.999997|0.774716|0.999960|0.000040| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.971001|0.537937|0.999999|0.768968|0.999961|0.000039| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.907196|0.432298|0.999996|0.716147|0.999950|0.000050| @@ -261,7 +253,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.985968|0.965783|0.946408|0.933831|0.966346|0.966343| |2|GERMAN CISTEM|PRIMARY_OUTPUT|0.878883|0.859676|0.841289|0.753887|0.860246|0.860236| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.830217|0.720244|0.635999|0.562799|0.738466|0.738450| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|0.830220|0.720249|0.636004|0.562804|0.738469|0.738454| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|0.817997|0.691285|0.598564|0.528217|0.715543|0.715527| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|0.836342|0.692324|0.590620|0.529431|0.722729|0.722714| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|0.743781|0.585563|0.482850|0.413990|0.626242|0.626223| @@ -276,7 +268,7 @@ This mode contains **12 result rows**, **8 evaluated stemmers**, and **3 output |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|801691|80|56726|10594963454|80 / 10594963534|56726 / 858417| |2|GERMAN CISTEM|PRIMARY_OUTPUT|712025|86055|146392|10594877479|86055 / 10594963534|146392 / 858417| -|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|506459|41477|351958|10594922057|41477 / 10594963534|351958 / 858417| +|3|SNOWBALL GERMAN DIRECT|PRIMARY_OUTPUT|506464|41477|351953|10594922057|41477 / 10594963534|351953 / 858417| |4|SNOWBALL GERMAN LUCENE FILTER|PRIMARY_OUTPUT|471644|34482|386773|10594929052|34482 / 10594963534|386773 / 858417| |5|GERMAN LUCENE GERMAN LIGHT STEM FILTER|PRIMARY_OUTPUT|461774|13791|396643|10594949743|13791 / 10594963534|396643 / 858417| |6|GERMAN LUCENE GERMAN STEM FILTER|PRIMARY_OUTPUT|371092|37962|487325|10594925572|37962 / 10594963534|487325 / 858417| @@ -378,7 +370,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `DE_DE` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/hebrew.md b/docs/benchmarks/languages/hebrew.md index d061353..6af6722 100644 --- a/docs/benchmarks/languages/hebrew.md +++ b/docs/benchmarks/languages/hebrew.md @@ -8,9 +8,9 @@ The default Hebrew model currently has no same-language third-party adapter in t ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `he-il-default` | `1.0.0` | `HE_IL` | 2,358 | 61,071 | 4,715 | 56,356 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `he-il-default` | `1.0.0` | `HE_IL` | 2,358 | 61,071 | 4,715 | 56,356 | 56,356 | ## Radixor Patch Command Distribution @@ -32,23 +32,17 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | --- | ---: | ---: | ---: | --- | | Radixor | 98.228% | 98.172% | 98.897% | Full default-model Radixor dictionary patch-command stemmer. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `hebrewRadixor` | 3.921 | 0.140 | 69.6 | 1.000 | Full default-model Radixor dictionary patch-command stemmer. | - - - +| Radixor | `hebrewRadixor` | 3.570 | 0.074 | 63.3 | 1.000 | Full default-model Radixor dictionary patch-command stemmer. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the default language model used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Hebrew patch commands use forward traversal as declared by the model metadata. - Results are environment-specific and should be compared only with rows from the same benchmark run. @@ -290,7 +284,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `HE_IL` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/hungarian.md b/docs/benchmarks/languages/hungarian.md index 764a97c..e3ad6f5 100644 --- a/docs/benchmarks/languages/hungarian.md +++ b/docs/benchmarks/languages/hungarian.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `hu-hu-default` | `1.0.0` | `HU_HU` | 19,406 | 935,713 | 38,775 | 896,938 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `hu-hu-default` | `1.0.0` | `HU_HU` | 19,406 | 935,713 | 38,775 | 896,938 | 896,938 | ## Radixor Patch Command Distribution @@ -30,13 +30,10 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.222% | 99.537% | 91.948% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 99.222% | 99.537% | 91.948% | Radixor dictionary-trained patch-command stemmer. | | Lucene SnowballFilter | 66.445% | 66.938% | 55.043% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 66.445% | 66.938% | 55.043% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | -| Lucene HungarianLightStemFilter | 14.748% | 14.777% | 14.086% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | - - - +| Lucene HungarianLightStemFilter | 14.748% | 14.777% | 14.086% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | ## Speed @@ -44,17 +41,14 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `hungarianRadixor` | 61.205 | 0.944 | 68.2 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HungarianLightStemFilter | `hungarianLuceneHungarianLightStemFilter` | 92.090 | 3.410 | 102.7 | 1.505 | Light Hungarian suffix stemmer. | -| Official Snowball direct | `snowballDirect[HUNGARIAN]` | 152.969 | 4.468 | 170.5 | 2.499 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[HUNGARIAN]` | 188.807 | 5.290 | 210.5 | 3.085 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `hungarianRadixor` | 52.020 | 1.337 | 58.0 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HungarianLightStemFilter | `hungarianLuceneHungarianLightStemFilter` | 87.362 | 3.444 | 97.4 | 1.679 | Light Hungarian suffix stemmer. | +| Official Snowball direct | `snowballDirect[HUNGARIAN]` | 158.081 | 9.757 | 176.2 | 3.039 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[HUNGARIAN]` | 179.398 | 8.041 | 200.0 | 3.449 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -322,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `HU_HU` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/italian.md b/docs/benchmarks/languages/italian.md index 2d90f7f..84529da 100644 --- a/docs/benchmarks/languages/italian.md +++ b/docs/benchmarks/languages/italian.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `it-it-default` | `1.0.0` | `IT_IT` | 10,009 | 337,546 | 20,004 | 317,542 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `it-it-default` | `1.0.0` | `IT_IT` | 10,009 | 337,546 | 20,004 | 317,542 | 317,542 | ## Radixor Patch Command Distribution @@ -29,31 +29,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.056% | 98.997% | 100.000% | Full Radixor dictionary patch-command stemmer. | -| Lucene ItalianLightStemFilter | 0.466% | 0.479% | 0.270% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Radixor | 99.056% | 98.997% | 100.000% | Radixor dictionary-trained patch-command stemmer. | +| Lucene ItalianLightStemFilter | 0.466% | 0.479% | 0.270% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | | Lucene SnowballFilter | 0.041% | 0.043% | 0.010% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 0.041% | 0.043% | 0.010% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `italianRadixor` | 25.073 | 0.534 | 79.0 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene ItalianLightStemFilter | `italianLuceneItalianLightStemFilter` | 15.956 | 0.184 | 50.2 | 0.636 | Light Italian suffix stemmer. | -| Official Snowball direct | `snowballDirect[ITALIAN]` | 115.818 | 3.174 | 364.7 | 4.619 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[ITALIAN]` | 123.974 | 4.405 | 390.4 | 4.944 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `italianRadixor` | 22.503 | 0.434 | 70.9 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene ItalianLightStemFilter | `italianLuceneItalianLightStemFilter` | 15.008 | 0.278 | 47.3 | 0.667 | Light Italian suffix stemmer. | +| Official Snowball direct | `snowballDirect[ITALIAN]` | 109.401 | 2.983 | 344.5 | 4.862 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[ITALIAN]` | 116.392 | 3.271 | 366.5 | 5.172 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -321,7 +315,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `IT_IT` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/norwegian-bokmal.md b/docs/benchmarks/languages/norwegian-bokmal.md index 44554b1..878c1b8 100644 --- a/docs/benchmarks/languages/norwegian-bokmal.md +++ b/docs/benchmarks/languages/norwegian-bokmal.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `nb-no-default` | `1.0.0` | `NB_NO` | 17,929 | 90,757 | 33,376 | 57,381 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `nb-no-default` | `1.0.0` | `NB_NO` | 17,929 | 90,757 | 33,376 | 57,381 | 57,381 | ## Radixor Patch Command Distribution @@ -30,14 +30,11 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 96.852% | 97.637% | 95.503% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 96.852% | 97.637% | 95.503% | Radixor dictionary-trained patch-command stemmer. | | Lucene NorwegianMinimalStemFilter | 57.107% | 53.913% | 62.599% | Minimal suffix reducer; narrow baseline, not a full stemmer. | | Official Snowball direct | 54.824% | 51.791% | 60.040% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene SnowballFilter | 54.803% | 51.780% | 60.001% | Lucene TokenFilter integration path around the Snowball algorithm. | -| Lucene NorwegianLightStemFilter | 52.136% | 50.616% | 54.749% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | - - - +| Lucene NorwegianLightStemFilter | 52.136% | 50.616% | 54.749% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | ## Speed @@ -45,18 +42,15 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `norwegianBokmalRadixor` | 3.401 | 0.055 | 59.3 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene NorwegianMinimalStemFilter | `norwegianBokmalLuceneNorwegianMinimalStemFilter` | 2.943 | 0.023 | 51.3 | 0.865 | Minimal Norwegian suffix reducer. | -| Lucene NorwegianLightStemFilter | `norwegianBokmalLuceneNorwegianLightStemFilter` | 3.358 | 0.036 | 58.5 | 0.987 | Light Norwegian suffix stemmer. | -| Official Snowball direct | `snowballDirect[NORWEGIAN_BOKMAL]` | 4.378 | 0.295 | 76.3 | 1.287 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_BOKMAL]` | 6.114 | 0.436 | 106.5 | 1.797 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `norwegianBokmalRadixor` | 3.240 | 0.087 | 56.5 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene NorwegianMinimalStemFilter | `norwegianBokmalLuceneNorwegianMinimalStemFilter` | 2.726 | 0.022 | 47.5 | 0.841 | Minimal Norwegian suffix reducer. | +| Lucene NorwegianLightStemFilter | `norwegianBokmalLuceneNorwegianLightStemFilter` | 3.136 | 0.028 | 54.6 | 0.968 | Light Norwegian suffix stemmer. | +| Official Snowball direct | `snowballDirect[NORWEGIAN_BOKMAL]` | 4.711 | 0.410 | 82.1 | 1.454 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_BOKMAL]` | 5.681 | 0.206 | 99.0 | 1.753 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -332,7 +326,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `NB_NO` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/norwegian-nynorsk.md b/docs/benchmarks/languages/norwegian-nynorsk.md index 87f4386..645a59a 100644 --- a/docs/benchmarks/languages/norwegian-nynorsk.md +++ b/docs/benchmarks/languages/norwegian-nynorsk.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `nn-no-default` | `1.0.0` | `NN_NO` | 4,688 | 19,651 | 6,089 | 13,562 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `nn-no-default` | `1.0.0` | `NN_NO` | 4,688 | 19,651 | 6,089 | 13,562 | 13,562 | ## Radixor Patch Command Distribution @@ -30,29 +30,23 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 93.089% | 91.395% | 96.863% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 93.089% | 91.395% | 96.863% | Radixor dictionary-trained patch-command stemmer. | | Official Snowball direct | 60.974% | 60.212% | 62.670% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene SnowballFilter | 60.918% | 60.146% | 62.638% | Lucene TokenFilter integration path around the Snowball algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `radixor[NORWEGIAN_NYNORSK]` | 0.617 | 0.062 | 45.5 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Official Snowball direct | `snowballDirect[NORWEGIAN_NYNORSK]` | 0.955 | 0.076 | 70.4 | 1.548 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_NYNORSK]` | 1.352 | 0.106 | 99.7 | 2.191 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `radixor[NORWEGIAN_NYNORSK]` | 0.584 | 0.057 | 43.1 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Official Snowball direct | `snowballDirect[NORWEGIAN_NYNORSK]` | 1.087 | 0.097 | 80.2 | 1.861 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_NYNORSK]` | 1.256 | 0.095 | 92.6 | 2.149 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -312,7 +306,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `NN_NO` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/persian.md b/docs/benchmarks/languages/persian.md index 8d45620..f3cc5af 100644 --- a/docs/benchmarks/languages/persian.md +++ b/docs/benchmarks/languages/persian.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `fa-ir-default` | `1.0.0` | `FA_IR` | 69 | 3,770 | 138 | 3,632 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `fa-ir-default` | `1.0.0` | `FA_IR` | 69 | 3,770 | 138 | 3,632 | 5,000 | ## Radixor Patch Command Distribution @@ -28,11 +28,9 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 95.836% | 95.677% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 95.836% | 95.677% | 100.000% | Radixor dictionary-trained patch-command stemmer. | | Lucene PersianStemFilter | 1.485% | 0.000% | 40.580% | Lucene Persian suffix stemmer with required normalization in the measured path. | - - - +| Official Snowball direct | 3.660% | 0.000% | 100.000% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. | ## Speed @@ -40,15 +38,13 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `persianRadixor` | 0.243 | 0.004 | 66.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene PersianStemFilter | `persianLucenePersianStemFilter` | 0.469 | 0.007 | 129.1 | 1.930 | Persian suffix stemmer with Lucene normalization in the measured path. | - - - +| Radixor | `persianRadixor` | 0.230 | 0.003 | 46.0 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene PersianStemFilter | `persianLucenePersianStemFilter` | 0.448 | 0.009 | 89.5 | 1.948 | Persian suffix stemmer with Lucene normalization in the measured path. | +| Official Snowball direct | `snowballDirect[PERSIAN]` | 1.490 | 0.055 | 298.1 | 6.486 | Official Snowball 3.1.0 generated Java stemmer; direct API. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -66,11 +62,11 @@ Runtime performance and linguistic grouping quality are independent dimensions. The default model is `fa-ir-default`, loaded from classpath resource `org/egothor/stemmer/models/fa-ir-default/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses. -- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.976360** among 2 deterministic stemmers. The runner-up is `PERSIAN LUCENE PERSIAN STEM FILTER` at 0.502212, a difference of 0.474148. This rank does not imply leadership in throughput or every secondary metric. -- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.976360** among 2 deterministic stemmers. The runner-up is `PERSIAN LUCENE PERSIAN STEM FILTER` at 0.502212, a difference of 0.474148. This rank does not imply leadership in throughput or every secondary metric. +- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.976360** among 3 deterministic stemmers. The runner-up is `SNOWBALL PERSIAN DIRECT` at 0.535123, a difference of 0.441236. This rank does not imply leadership in throughput or every secondary metric. +- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.976360** among 3 deterministic stemmers. The runner-up is `SNOWBALL PERSIAN DIRECT` at 0.535123, a difference of 0.441236. This rank does not imply leadership in throughput or every secondary metric. ### `ALL_WORDS` -This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -79,7 +75,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) | |---:|---|---:|---:|---:| |1|Radixor|0.976360|0.000000%|4.728041%| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%| +|2|SNOWBALL PERSIAN DIRECT|0.535123|0.001278%|92.974054%| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%| @@ -88,7 +85,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|1.000000|0.952720|1.000000|0.976360|0.999277|0.000723| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.988428|0.070259|0.999987|0.535123|0.985764|0.014236| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231| @@ -97,7 +95,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.990172|0.975787|0.961815|0.952720|0.976074|0.975715| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.273526|0.131193|0.086291|0.070202|0.263527|0.261598| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774| @@ -106,7 +105,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|91503|0|4541|6182152|0 / 6182152|4541 / 96044| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|6748|79|89296|6182073|79 / 6182152|89296 / 96044| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044| @@ -174,7 +174,7 @@ Alternative candidates are capability analyses, not replacements for the determi ### `LOWERCASE_GROUPS_ONLY` -This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **5 result rows**, **3 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -183,7 +183,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) | |---:|---|---:|---:|---:| |1|Radixor|0.976360|0.000000%|4.728041%| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%| +|2|SNOWBALL PERSIAN DIRECT|0.535123|0.001278%|92.974054%| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|0.502212|0.000049%|99.557494%| @@ -192,7 +193,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|1.000000|0.952720|1.000000|0.976360|0.999277|0.000723| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.988428|0.070259|0.999987|0.535123|0.985764|0.014236| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.992991|0.004425|1.000000|0.502212|0.984769|0.015231| @@ -201,7 +203,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | F0.5 | F1 | F2 | Jaccard | Fowlkes–Mallows | MCC | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|0.990172|0.975787|0.961815|0.952720|0.976074|0.975715| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|0.273526|0.131193|0.086291|0.070202|0.263527|0.261598| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|0.021738|0.008811|0.005525|0.004425|0.066288|0.065774| @@ -210,7 +213,8 @@ This mode contains **4 result rows**, **2 evaluated stemmers**, and **3 output p | Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible | |---:|---|---|---:|---:|---:|---:|---:|---:| |1|Radixor|PRIMARY_OUTPUT|91503|0|4541|6182152|0 / 6182152|4541 / 96044| -|2|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044| +|2|SNOWBALL PERSIAN DIRECT|PRIMARY_OUTPUT|6748|79|89296|6182073|79 / 6182152|89296 / 96044| +|3|PERSIAN LUCENE PERSIAN STEM FILTER|PRIMARY_OUTPUT|425|3|95619|6182149|3 / 6182152|95619 / 96044| @@ -300,7 +304,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `FA_IR` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/polish.md b/docs/benchmarks/languages/polish.md index 383cac9..f60cf12 100644 --- a/docs/benchmarks/languages/polish.md +++ b/docs/benchmarks/languages/polish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `pl-pl-unimorph` | `1.0.0` | `PL_PL` | 9,990 | 132,308 | 19,957 | 112,351 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `pl-pl-unimorph` | `1.0.0` | `PL_PL` | 9,990 | 132,308 | 19,957 | 112,351 | 112,351 | ## Radixor Patch Command Distribution @@ -30,14 +30,12 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 98.837% | 98.744% | 99.359% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 98.837% | 98.744% | 99.359% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 89.545% | 88.272% | 96.713% | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene MorfologikFilter | 87.729% | 86.606% | 94.047% | Dictionary-based path; Morfologik can emit multiple terms. | | Lucene StempelFilter | 70.009% | 69.262% | 74.220% | Lucene TokenFilter integration path for table-driven Polish Stempel. | | Lucene StempelStemmer direct | 70.009% | 69.262% | 74.220% | Direct table-driven Polish Stempel stemmer API. | - - - +| Official Snowball direct | 22.315% | 20.225% | 34.078% | Official Snowball 3.1.0 generated Java stemmer; rule-based suffix algorithm. | ## Speed @@ -45,18 +43,16 @@ Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 i | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `polishRadixor` | 8.972 | 0.203 | 79.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 524.081 | 35.121 | 4664.7 | 58.412 | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Lucene StempelStemmer direct | `polishLuceneStempelStemmerDirect` | 37.947 | 0.335 | 337.8 | 4.229 | Direct table-driven Polish Stempel stemmer API. | -| Lucene StempelFilter | `polishLuceneStempelFilter` | 43.090 | 0.411 | 383.5 | 4.803 | Lucene TokenFilter integration path for table-driven Polish Stempel. | -| Lucene MorfologikFilter | `polishLuceneMorfologikFilter` | 143.527 | 1.176 | 1277.5 | 15.997 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | - - - +| Radixor | `polishRadixor` | 8.122 | 0.146 | 72.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 471.669 | 26.993 | 4198.2 | 58.070 | Benchmark-only Polish Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Lucene StempelStemmer direct | `polishLuceneStempelStemmerDirect` | 31.524 | 0.189 | 280.6 | 3.881 | Direct table-driven Polish Stempel stemmer API. | +| Lucene StempelFilter | `polishLuceneStempelFilter` | 39.180 | 0.362 | 348.7 | 4.824 | Lucene TokenFilter integration path for table-driven Polish Stempel. | +| Lucene MorfologikFilter | `polishLuceneMorfologikFilter` | 138.971 | 1.429 | 1236.9 | 17.110 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | +| Official Snowball direct | `snowballDirect[POLISH]` | 9.715 | 0.858 | 86.5 | 1.196 | Official Snowball 3.1.0 generated Java stemmer; direct API. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -74,11 +70,11 @@ Runtime performance and linguistic grouping quality are independent dimensions. The default model is `pl-pl-unimorph`, loaded from classpath resource `org/egothor/stemmer/models/pl-pl-unimorph/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses. -- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.991105** among 5 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948392, a difference of 0.042713. This rank does not imply leadership in throughput or every secondary metric. -- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.991301** among 5 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948417, a difference of 0.042884. This rank does not imply leadership in throughput or every secondary metric. +- **ALL_WORDS:** `Radixor` ranks first by balanced accuracy at **0.991105** among 6 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948392, a difference of 0.042713. This rank does not imply leadership in throughput or every secondary metric. +- **LOWERCASE_GROUPS_ONLY:** `Radixor` ranks first by balanced accuracy at **0.991301** among 6 deterministic stemmers. The runner-up is `POLISH LUCENE MORFOLOGIK FILTER` at 0.948417, a difference of 0.042884. This rank does not imply leadership in throughput or every secondary metric. ### `ALL_WORDS` -This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **12 result rows**, **6 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -91,6 +87,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|0.933457|0.000383%|13.308172%| |4|POLISH LUCENE STEMPEL DIRECT|0.855699|0.000602%|28.859618%| |5|POLISH LUCENE STEMPEL FILTER|0.855699|0.000602%|28.859618%| +|6|SNOWBALL POLISH DIRECT|0.823625|0.000967%|35.273970%| @@ -103,6 +100,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.971931|0.866918|0.999996|0.933457|0.999976|0.000024| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.947549|0.711404|0.999994|0.855699|0.999950|0.000050| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.947549|0.711404|0.999994|0.855699|0.999950|0.000050| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.910978|0.647260|0.999990|0.823625|0.999936|0.000064| @@ -115,6 +113,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.948942|0.916426|0.886065|0.845744|0.917924|0.917913| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.888559|0.812669|0.748723|0.684450|0.821030|0.821007| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.888559|0.812669|0.748723|0.684450|0.821030|0.821007| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.842338|0.756803|0.687038|0.608756|0.767880|0.767852| @@ -127,6 +126,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|968411|27967|148662|7303210371|27967 / 7303238338|148662 / 1117073| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|794690|43990|322383|7303194348|43990 / 7303238338|322383 / 1117073| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|794690|43990|322383|7303194348|43990 / 7303238338|322383 / 1117073| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|723037|70656|394036|7303167682|70656 / 7303238338|394036 / 1117073| @@ -208,7 +208,7 @@ Alternative candidates are capability analyses, not replacements for the determi ### `LOWERCASE_GROUPS_ONLY` -This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. +This mode contains **12 result rows**, **6 evaluated stemmers**, and **3 output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score. #### `PRIMARY_OUTPUT` ranking @@ -221,6 +221,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|0.933546|0.000382%|13.290396%| |4|POLISH LUCENE STEMPEL DIRECT|0.856335|0.000611%|28.732387%| |5|POLISH LUCENE STEMPEL FILTER|0.856335|0.000611%|28.732387%| +|6|SNOWBALL POLISH DIRECT|0.823465|0.000990%|35.306102%| @@ -233,6 +234,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.972469|0.867096|0.999996|0.933546|0.999975|0.000025| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.947796|0.712676|0.999994|0.856335|0.999949|0.000051| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.947796|0.712676|0.999994|0.856335|0.999949|0.000051| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.910487|0.646939|0.999990|0.823465|0.999935|0.000065| @@ -245,6 +247,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|0.949394|0.916764|0.886303|0.846320|0.918272|0.918260| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|0.889130|0.813590|0.749881|0.685758|0.821871|0.821848| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|0.889130|0.813590|0.749881|0.685758|0.821871|0.821848| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|0.841894|0.756414|0.686693|0.608253|0.767483|0.767454| @@ -257,6 +260,7 @@ This mode contains **11 result rows**, **5 evaluated stemmers**, and **3 output |3|HUNSPELL POLISH LUCENE FILTER|PRIMARY_OUTPUT|963133|27267|147624|7133072951|27267 / 7133100218|147624 / 1110757| |4|POLISH LUCENE STEMPEL DIRECT|PRIMARY_OUTPUT|791610|43601|319147|7133056617|43601 / 7133100218|319147 / 1110757| |5|POLISH LUCENE STEMPEL FILTER|PRIMARY_OUTPUT|791610|43601|319147|7133056617|43601 / 7133100218|319147 / 1110757| +|6|SNOWBALL POLISH DIRECT|PRIMARY_OUTPUT|718592|70647|392165|7133029571|70647 / 7133100218|392165 / 1110757| @@ -360,7 +364,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `PL_PL` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/portuguese.md b/docs/benchmarks/languages/portuguese.md index a1e2dd5..c1d440e 100644 --- a/docs/benchmarks/languages/portuguese.md +++ b/docs/benchmarks/languages/portuguese.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `pt-pt-default` | `1.0.0` | `PT_PT` | 4,001 | 215,490 | 8,002 | 207,488 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `pt-pt-default` | `1.0.0` | `PT_PT` | 4,001 | 215,490 | 8,002 | 207,488 | 207,488 | ## Radixor Patch Command Distribution @@ -30,35 +30,29 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.815% | 99.808% | 100.000% | Full Radixor dictionary patch-command stemmer. | -| Lucene PortugueseLightStemFilter | 8.966% | 5.558% | 97.326% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Radixor | 99.815% | 99.808% | 100.000% | Radixor dictionary-trained patch-command stemmer. | +| Lucene PortugueseLightStemFilter | 8.966% | 5.558% | 97.326% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | | Lucene PortugueseMinimalStemFilter | 5.539% | 1.896% | 100.000% | Minimal suffix reducer; narrow baseline, not a full stemmer. | | Lucene SnowballFilter | 0.625% | 0.558% | 2.374% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 0.625% | 0.558% | 2.374% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene PortugueseStemFilter | 0.312% | 0.308% | 0.425% | Portuguese RSLP-style Lucene TokenFilter stemmer. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `portugueseRadixor` | 12.301 | 0.252 | 59.3 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene PortugueseLightStemFilter | `portugueseLucenePortugueseLightStemFilter` | 11.409 | 0.151 | 55.0 | 0.927 | Light Portuguese suffix stemmer. | -| Lucene PortugueseMinimalStemFilter | `portugueseLucenePortugueseMinimalStemFilter` | 15.619 | 0.084 | 75.3 | 1.270 | Minimal Portuguese suffix reducer. | -| Official Snowball direct | `snowballDirect[PORTUGUESE]` | 57.577 | 1.591 | 277.5 | 4.681 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[PORTUGUESE]` | 63.403 | 2.720 | 305.6 | 5.154 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | -| Lucene PortugueseStemFilter | `portugueseLucenePortugueseStemFilter` | 158.014 | 5.150 | 761.6 | 12.845 | Portuguese RSLP-style Lucene TokenFilter. | - - - +| Radixor | `portugueseRadixor` | 10.902 | 0.166 | 52.5 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene PortugueseLightStemFilter | `portugueseLucenePortugueseLightStemFilter` | 10.125 | 0.161 | 48.8 | 0.929 | Light Portuguese suffix stemmer. | +| Lucene PortugueseMinimalStemFilter | `portugueseLucenePortugueseMinimalStemFilter` | 14.338 | 0.156 | 69.1 | 1.315 | Minimal Portuguese suffix reducer. | +| Official Snowball direct | `snowballDirect[PORTUGUESE]` | 52.193 | 1.905 | 251.5 | 4.788 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[PORTUGUESE]` | 58.991 | 2.457 | 284.3 | 5.411 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | +| Lucene PortugueseStemFilter | `portugueseLucenePortugueseStemFilter` | 164.457 | 8.688 | 792.6 | 15.085 | Portuguese RSLP-style Lucene TokenFilter. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -342,7 +336,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `PT_PT` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/russian.md b/docs/benchmarks/languages/russian.md index 058d3f3..a2366ae 100644 --- a/docs/benchmarks/languages/russian.md +++ b/docs/benchmarks/languages/russian.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `ru-ru-default` | `1.0.0` | `RU_RU` | 37,410 | 806,279 | 74,808 | 731,471 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `ru-ru-default` | `1.0.0` | `RU_RU` | 37,410 | 806,279 | 74,808 | 731,471 | 731,471 | ## Radixor Patch Command Distribution @@ -30,31 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 98.807% | 98.696% | 99.896% | Full Radixor dictionary patch-command stemmer. | -| Lucene RussianLightStemFilter | 9.658% | 8.452% | 21.447% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Radixor | 98.807% | 98.696% | 99.896% | Radixor dictionary-trained patch-command stemmer. | +| Lucene RussianLightStemFilter | 9.658% | 8.452% | 21.447% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | | Lucene SnowballFilter | 9.162% | 8.162% | 18.936% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 9.162% | 8.162% | 18.936% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `russianRadixor` | 90.151 | 1.796 | 123.2 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene RussianLightStemFilter | `russianLuceneRussianLightStemFilter` | 59.456 | 2.102 | 81.3 | 0.660 | Light Russian suffix stemmer. | -| Official Snowball direct | `snowballDirect[RUSSIAN]` | 102.353 | 1.669 | 139.9 | 1.135 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[RUSSIAN]` | 138.597 | 4.727 | 189.5 | 1.537 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `russianRadixor` | 72.723 | 1.809 | 99.4 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene RussianLightStemFilter | `russianLuceneRussianLightStemFilter` | 58.844 | 3.217 | 80.4 | 0.809 | Light Russian suffix stemmer. | +| Official Snowball direct | `snowballDirect[RUSSIAN]` | 103.471 | 8.136 | 141.5 | 1.423 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[RUSSIAN]` | 130.783 | 3.979 | 178.8 | 1.798 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -322,7 +316,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `RU_RU` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/spanish.md b/docs/benchmarks/languages/spanish.md index 5f820af..77f3212 100644 --- a/docs/benchmarks/languages/spanish.md +++ b/docs/benchmarks/languages/spanish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `es-es-default` | `1.0.0` | `ES_ES` | 65,059 | 926,393 | 120,121 | 806,272 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `es-es-default` | `1.0.0` | `ES_ES` | 65,059 | 926,393 | 120,121 | 806,272 | 806,272 | ## Radixor Patch Command Distribution @@ -30,37 +30,31 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 97.459% | 97.544% | 96.891% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 97.459% | 97.544% | 96.891% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 49.074% | 42.656% | 92.154% | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene SpanishMinimalStemFilter | 17.284% | 5.347% | 97.403% | Minimal suffix reducer; narrow baseline, not a full stemmer. | | Lucene SpanishPluralStemFilter | 15.140% | 5.802% | 77.820% | Plural-focused suffix reducer; narrow baseline. | -| Lucene SpanishLightStemFilter | 9.577% | 7.088% | 26.279% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene SpanishLightStemFilter | 9.577% | 7.088% | 26.279% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | | Lucene SnowballFilter | 4.889% | 4.287% | 8.932% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 4.889% | 4.287% | 8.930% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `spanishRadixor` | 81.605 | 1.347 | 101.2 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 2033.430 | 14.863 | 2522.0 | 24.918 | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Lucene SpanishMinimalStemFilter | `spanishLuceneSpanishMinimalStemFilter` | 42.144 | 1.556 | 52.3 | 0.516 | Minimal Spanish suffix reducer; narrow baseline. | -| Lucene SpanishLightStemFilter | `spanishLuceneSpanishLightStemFilter` | 44.479 | 1.291 | 55.2 | 0.545 | Light Spanish suffix stemmer. | -| Lucene SpanishPluralStemFilter | `spanishLuceneSpanishPluralStemFilter` | 96.537 | 3.418 | 119.7 | 1.183 | Plural-oriented Spanish suffix reducer. | -| Official Snowball direct | `snowballDirect[SPANISH]` | 172.151 | 7.261 | 213.5 | 2.110 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[SPANISH]` | 201.697 | 9.363 | 250.2 | 2.472 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `spanishRadixor` | 63.164 | 1.885 | 78.3 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 1936.800 | 18.685 | 2402.2 | 30.663 | Benchmark-only Spanish Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Lucene SpanishMinimalStemFilter | `spanishLuceneSpanishMinimalStemFilter` | 40.414 | 1.475 | 50.1 | 0.640 | Minimal Spanish suffix reducer; narrow baseline. | +| Lucene SpanishLightStemFilter | `spanishLuceneSpanishLightStemFilter` | 43.922 | 1.497 | 54.5 | 0.695 | Light Spanish suffix stemmer. | +| Lucene SpanishPluralStemFilter | `spanishLuceneSpanishPluralStemFilter` | 89.799 | 3.474 | 111.4 | 1.422 | Plural-oriented Spanish suffix reducer. | +| Official Snowball direct | `snowballDirect[SPANISH]` | 192.868 | 11.684 | 239.2 | 3.053 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[SPANISH]` | 182.719 | 6.957 | 226.6 | 2.893 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -366,7 +360,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `ES_ES` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/swedish.md b/docs/benchmarks/languages/swedish.md index d686779..750126c 100644 --- a/docs/benchmarks/languages/swedish.md +++ b/docs/benchmarks/languages/swedish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `sv-se-default` | `1.0.0` | `SV_SE` | 12,371 | 110,468 | 24,731 | 85,737 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `sv-se-default` | `1.0.0` | `SV_SE` | 12,371 | 110,468 | 24,731 | 85,737 | 85,737 | ## Radixor Patch Command Distribution @@ -30,33 +30,27 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 96.713% | 97.407% | 94.307% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 96.713% | 97.407% | 94.307% | Radixor dictionary-trained patch-command stemmer. | | Lucene SwedishMinimalStemFilter | 49.532% | 49.186% | 50.730% | Minimal suffix reducer; narrow baseline, not a full stemmer. | -| Lucene SwedishLightStemFilter | 45.672% | 46.383% | 43.209% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene SwedishLightStemFilter | 45.672% | 46.383% | 43.209% | Light suffix stemmer; intentionally narrower than Radixor's dictionary-trained transformation model. | | Official Snowball direct | 40.068% | 37.512% | 48.926% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | | Lucene SnowballFilter | 38.785% | 35.839% | 48.999% | Lucene TokenFilter integration path around the Snowball algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `swedishRadixor` | 5.476 | 0.081 | 63.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene SwedishMinimalStemFilter | `swedishLuceneSwedishMinimalStemFilter` | 4.741 | 0.086 | 55.3 | 0.866 | Minimal Swedish suffix reducer. | -| Lucene SwedishLightStemFilter | `swedishLuceneSwedishLightStemFilter` | 4.893 | 0.053 | 57.1 | 0.893 | Light Swedish suffix stemmer. | -| Official Snowball direct | `snowballDirect[SWEDISH]` | 7.606 | 0.555 | 88.7 | 1.389 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[SWEDISH]` | 10.295 | 0.749 | 120.1 | 1.880 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `swedishRadixor` | 5.078 | 0.104 | 59.2 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene SwedishMinimalStemFilter | `swedishLuceneSwedishMinimalStemFilter` | 4.417 | 0.061 | 51.5 | 0.870 | Minimal Swedish suffix reducer. | +| Lucene SwedishLightStemFilter | `swedishLuceneSwedishLightStemFilter` | 5.090 | 0.373 | 59.4 | 1.002 | Light Swedish suffix stemmer. | +| Official Snowball direct | `snowballDirect[SWEDISH]` | 7.497 | 0.653 | 87.4 | 1.476 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[SWEDISH]` | 9.831 | 0.648 | 114.7 | 1.936 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -332,7 +326,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `SV_SE` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/ukrainian.md b/docs/benchmarks/languages/ukrainian.md index a6dee0b..d362bce 100644 --- a/docs/benchmarks/languages/ukrainian.md +++ b/docs/benchmarks/languages/ukrainian.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `uk-ua-default` | `1.0.0` | `UK_UA` | 1,493 | 15,737 | 2,985 | 12,752 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `uk-ua-default` | `1.0.0` | `UK_UA` | 1,493 | 15,737 | 2,985 | 12,752 | 12,752 | ## Radixor Patch Command Distribution @@ -30,31 +30,25 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 99.307% | 99.365% | 99.062% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 99.307% | 99.365% | 99.062% | Radixor dictionary-trained patch-command stemmer. | | Lucene HunspellStemFilter | 86.815% | 83.759% | 99.866% | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. | | Lucene MorfologikFilter | 92.362% | 90.637% | 99.732% | Dictionary-based path; Morfologik can emit multiple terms. | | Morfologik direct | 92.362% | 90.637% | 99.732% | Direct dictionary lookup; first returned stem is used for quality when no ranking weight is exposed. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `ukrainianRadixor` | 0.639 | 0.009 | 50.1 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 47.919 | 5.308 | 3757.8 | 74.957 | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. | -| Morfologik direct | `ukrainianMorfologikDirect` | 8.662 | 0.105 | 679.3 | 13.550 | Direct Morfologik dictionary lookup; first returned stem is used for quality. | -| Lucene MorfologikFilter | `ukrainianLuceneMorfologikFilter` | 15.367 | 0.219 | 1205.1 | 24.038 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | - - - +| Radixor | `ukrainianRadixor` | 0.594 | 0.010 | 46.6 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Lucene HunspellStemFilter | `luceneHunspellStemFilter` | 39.820 | 3.772 | 3122.6 | 67.067 | Benchmark-only Ukrainian Hunspell dictionary compared via Lucene HunspellStemFilter. | +| Morfologik direct | `ukrainianMorfologikDirect` | 8.231 | 0.121 | 645.5 | 13.863 | Direct Morfologik dictionary lookup; first returned stem is used for quality. | +| Lucene MorfologikFilter | `ukrainianLuceneMorfologikFilter` | 14.700 | 0.176 | 1152.8 | 24.758 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -364,7 +358,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `UK_UA` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/languages/yiddish.md b/docs/benchmarks/languages/yiddish.md index 3647d1b..c477b81 100644 --- a/docs/benchmarks/languages/yiddish.md +++ b/docs/benchmarks/languages/yiddish.md @@ -8,9 +8,9 @@ Radixor must not be read as simply "slower" when a narrow competitor has a lower ## Dictionary Corpus -| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | -| --- | --- | --- | ---: | ---: | ---: | ---: | -| `yi-default` | `1.0.0` | `YI` | 802 | 4,300 | 1,524 | 2,776 | +| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | +| `yi-default` | `1.0.0` | `YI` | 802 | 4,300 | 1,524 | 2,776 | 5,000 | ## Radixor Patch Command Distribution @@ -29,29 +29,23 @@ Accuracy is computed from JMH auxiliary counters in the current report. The coun | Stemmer | All exact | Changed exact | Root preserved | Note | | --- | ---: | ---: | ---: | --- | -| Radixor | 98.930% | 98.343% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Radixor | 98.930% | 98.343% | 100.000% | Radixor dictionary-trained patch-command stemmer. | | Lucene SnowballFilter | 2.837% | 2.558% | 3.346% | Lucene TokenFilter integration path around the Snowball algorithm. | | Official Snowball direct | 2.837% | 2.558% | 3.346% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | - - - ## Speed Speed uses JMH average time, 5 warmup iterations, 10 measurement iterations, 3 independent forks, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. | Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | | --- | --- | ---: | ---: | ---: | ---: | --- | -| Radixor | `radixor[YIDDISH]` | 0.249 | 0.001 | 89.6 | 1.000 | Full Radixor dictionary patch-command stemmer. | -| Official Snowball direct | `snowballDirect[YIDDISH]` | 1.574 | 0.066 | 567.1 | 6.330 | Official Snowball generated Java stemmer; direct API. | -| Lucene SnowballFilter | `luceneSnowballFilter[YIDDISH]` | 1.849 | 0.079 | 665.9 | 7.434 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | - - - +| Radixor | `radixor[YIDDISH]` | 0.234 | 0.001 | 46.8 | 1.000 | Radixor dictionary-trained patch-command stemmer. | +| Official Snowball direct | `snowballDirect[YIDDISH]` | 1.487 | 0.062 | 297.5 | 6.354 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[YIDDISH]` | 1.754 | 0.070 | 350.8 | 7.492 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | ## Interpretation Notes -- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Radixor is a dictionary-trained patch-command stemmer. Its learned transformations can generalize beyond the word forms listed in the training resource. - Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. - Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. - Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. @@ -311,7 +305,7 @@ Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: ### Provenance - Authoritative source: `docs/benchmarks/data/stemming-quality.csv` -- Source SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` +- Source SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` - Evaluation command: `./gradlew stemmingQuality --no-daemon` - Dictionary language: `YI` - Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY` diff --git a/docs/benchmarks/reference/english-coverage.md b/docs/benchmarks/reference/english-coverage.md index 18dd028..2fc7f02 100644 --- a/docs/benchmarks/reference/english-coverage.md +++ b/docs/benchmarks/reference/english-coverage.md @@ -6,16 +6,16 @@ This benchmark is the clearest demonstration of the Radixor quality/speed envelo | Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token | | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 20.627 | 2.117 | 98.0 | -| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.713 | 2.104 | 103.2 | -| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 17.408 | 1.438 | 82.7 | -| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 16.946 | 1.531 | 80.5 | -| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 15.735 | 1.278 | 74.8 | -| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 14.714 | 1.089 | 69.9 | -| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.090 | 1.254 | 71.7 | -| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 13.773 | 1.071 | 65.4 | -| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 15.396 | 2.497 | 73.1 | -| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 16.970 | 2.847 | 80.6 | +| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 15.064 | 0.658 | 71.6 | +| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 17.798 | 2.161 | 84.6 | +| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 13.900 | 0.941 | 66.0 | +| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 14.809 | 1.376 | 70.3 | +| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 13.186 | 0.930 | 62.6 | +| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 12.852 | 0.943 | 61.1 | +| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 12.358 | 0.831 | 58.7 | +| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 11.657 | 0.921 | 55.4 | +| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 11.494 | 1.256 | 54.6 | +| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 9.895 | 0.925 | 47.0 | ## Column Meanings diff --git a/docs/benchmarks/reference/environment.md b/docs/benchmarks/reference/environment.md index aec3799..5bc3323 100644 --- a/docs/benchmarks/reference/environment.md +++ b/docs/benchmarks/reference/environment.md @@ -4,25 +4,25 @@ The values below are environment-specific and must not be read as universal perf | Item | Value | | --- | --- | -| Benchmark date | 2026-07-23 (Europe/Prague) | +| Benchmark date | 2026-08-10 (Europe/Prague) | | Corpus command | `./gradlew benchmarkCorpusReport --no-daemon` | -| Exact-root accuracy command | Direct JMH execution of the four `*BenchmarkQuality` classes selected in `stemmer-accuracy-2026-07-23.txt`; timing scores are discarded | +| Exact-root accuracy command | `tools/run-published-accuracy-benchmarks.sh 2026-08-10`; all four `*BenchmarkQuality` classes are selected and timing scores are discarded | | Stemming-quality command | `./gradlew stemmingQuality --no-daemon` | -| Published speed command | `tools/run-published-speed-benchmarks.sh 2026-07-23` | -| Published speed run interval | 2026-07-23 12:58:50 to 15:15:43 Europe/Prague (2 h 16 min 53 s, including idle intervals and both JMH suites) | +| Published speed command | `tools/run-published-speed-benchmarks.sh 2026-08-10` | +| Published speed run interval | 2026-08-10 16:22:33 to 18:36:41 Europe/Prague (2 h 14 min 8 s, including idle intervals and both JMH suites) | | Stabilization intervals | 120 s before the main speed matrix; 60 s between the main matrix and coverage-speed suite | | Corpus and command report | `build/reports/jmh/benchmark-corpora.csv` | -| Exact-root reports | `build/reports/jmh/stemmer-accuracy-2026-07-23.csv` and `.txt` | -| Speed reports | `build/reports/jmh/stemmer-speed-2026-07-23.csv` and `.txt` | -| English coverage accuracy reports | `build/reports/jmh/english-coverage-accuracy-2026-07-23.csv` and `.txt` | -| English coverage speed reports | `build/reports/jmh/english-coverage-speed-2026-07-23.csv` and `.txt` | +| Exact-root reports | `build/reports/jmh/stemmer-accuracy-2026-08-10.csv` and `.txt` | +| Speed reports | `build/reports/jmh/stemmer-speed-2026-08-10.csv` and `.txt` | +| English coverage accuracy reports | `build/reports/jmh/english-coverage-accuracy-2026-08-10.csv` and `.txt` | +| English coverage speed reports | `build/reports/jmh/english-coverage-speed-2026-08-10.csv` and `.txt` | | Stemming-quality reports | `build/reports/stemming-quality/stemming-quality.csv` and `.md` | -| Environment report | `build/reports/jmh/performance-environment-2026-07-23.txt` | -| Selected speed methods | `build/reports/jmh/published-speed-benchmarks-2026-07-23.txt` | +| Environment report | `build/reports/jmh/performance-environment-2026-08-10.txt` | +| Selected speed methods | `build/reports/jmh/published-speed-benchmarks-2026-08-10.txt` | | Comparison scope | Same-language methods used by the 20 language pages; `PolishPolimorfStemmerComparisonBenchmark`, all quality methods, the separate CISTEM gold-standard experiment, and internal trie microbenchmarks are excluded | | Model scope | Exactly the 20 IDs declared by `Language.defaultModelId()`; Polish uses `pl-pl-unimorph`, and `pl-pl-polimorf` is not measured | -| Core base commit | `1f1b03c6a8d36a0918b92ebde698e5379a2a5946` | -| Measured source state | `release@4.0.0-dirty`; exact tracked changes and untracked-source checksums are retained as `measured-source-2026-07-23.patch` and `measured-untracked-2026-07-23.sha256` | +| Core base commit | `b45e143c8484c2ae1d6e31069a2e67232c4f7f29` | +| Measured source state | `release@4.1.0-dirty`; exact tracked changes and untracked-source checksums are retained as `measured-source-2026-08-10.patch` and `measured-untracked-2026-08-10.sha256` | | JMH version | 1.37 | | Speed benchmark mode | Average time, `time/op` | | Score unit | `ns/op`; language pages additionally derive `ms/op` and `ns/token` | @@ -32,21 +32,21 @@ The values below are environment-specific and must not be read as universal perf | Speed threads | 1 | | Speed fork heap | Fixed `-Xms6g -Xmx6g` | | Reported uncertainty | JMH `Score Error (99.9%)` over 30 measured samples | -| Observed relative uncertainty | Main speed matrix: maximum 11.945%, with 6 of 102 rows above 10%; coverage-speed curve: maximum 16.775%, with 3 of 10 rows above 10%; no published row exceeded 20% | +| Observed relative uncertainty | Main speed matrix: maximum 10.607%, with 2 of 105 rows above 10%; coverage-speed curve: maximum 12.142%, with 2 of 10 rows above 10%; no published row exceeded 20% | | Deterministic measurements | Corpus, patch-command distribution, exact-root counters, coverage accuracy, and pairwise stemming quality are evaluated without interpreting runtime scores; no warmup is required | -| JVM reported by JMH | JDK 25.0.3, OpenJDK 64-Bit Server VM, 25.0.3+9 | -| Java runtime | OpenJDK Runtime Environment, Red Hat build 25.0.3+9 | +| JVM reported by JMH | JDK 25.0.4, OpenJDK 64-Bit Server VM, 25.0.4+7 | +| Java runtime | OpenJDK Runtime Environment, Red Hat build 25.0.4+7 | | JVM invoker | `/usr/lib/jvm/java-25-openjdk/bin/java` | | Operating system | Fedora Linux 44 (MATE-Compiz) | -| Kernel | Linux 7.1.4-200.fc44.x86_64 | +| Kernel | Linux 7.1.7-200.fc44.x86_64 | | Architecture | x86_64 | -| CPU | AMD Ryzen 5 8600G w/ Radeon 760M Graphics | +| CPU | AMD Ryzen 5 7600 6-Core Processor | | Physical / logical CPUs | 6 / 12 | | CPU frequency policy | `amd-pstate-epp`; governor `performance` on every logical CPU; EPP `performance`; boost enabled | | CPU affinity | Scheduler default; no explicit pinning | -| Installed memory | 60 GiB reported by the operating system | -| Pre-run idle state | Load average 0.25 / 0.36 / 0.71 after the 120 s idle interval; CPU Tctl 40.2 degrees Celsius; swap unused | -| End-of-run state | Load average 1.16 / 1.28 / 1.32; CPU Tctl 60.5 degrees Celsius | +| Installed memory | 61 GiB reported by the operating system | +| Pre-run idle state | Load average 0.16 / 0.42 / 0.88 after the 120 s idle interval; CPU Tctl 57.4 degrees Celsius; swap unused | +| End-of-run state | Load average 1.59 / 1.49 / 1.36; CPU Tctl 78.1 degrees Celsius | | Power and idle policy | Developer workstation on stable power; screensaver, suspend, and hibernation disabled | | Concurrent project work | None during the published speed and coverage-speed run | @@ -67,15 +67,15 @@ The JMH runtime classpath contains the optional model artifact because it is a s Generated local report files for this benchmark update: - `build/reports/jmh/benchmark-corpora.csv` -- `build/reports/jmh/stemmer-accuracy-2026-07-23.csv` -- `build/reports/jmh/stemmer-accuracy-2026-07-23.txt` -- `build/reports/jmh/stemmer-speed-2026-07-23.csv` -- `build/reports/jmh/stemmer-speed-2026-07-23.txt` -- `build/reports/jmh/english-coverage-accuracy-2026-07-23.csv` -- `build/reports/jmh/english-coverage-accuracy-2026-07-23.txt` -- `build/reports/jmh/english-coverage-speed-2026-07-23.csv` -- `build/reports/jmh/english-coverage-speed-2026-07-23.txt` -- `build/reports/jmh/performance-environment-2026-07-23.txt` +- `build/reports/jmh/stemmer-accuracy-2026-08-10.csv` +- `build/reports/jmh/stemmer-accuracy-2026-08-10.txt` +- `build/reports/jmh/stemmer-speed-2026-08-10.csv` +- `build/reports/jmh/stemmer-speed-2026-08-10.txt` +- `build/reports/jmh/english-coverage-accuracy-2026-08-10.csv` +- `build/reports/jmh/english-coverage-accuracy-2026-08-10.txt` +- `build/reports/jmh/english-coverage-speed-2026-08-10.csv` +- `build/reports/jmh/english-coverage-speed-2026-08-10.txt` +- `build/reports/jmh/performance-environment-2026-08-10.txt` - `build/reports/stemming-quality/stemming-quality.csv` - `build/reports/stemming-quality/stemming-quality.md` - `build/reports/stemming-quality/metric-correlations-pearson.csv` diff --git a/docs/benchmarks/reference/methodology.md b/docs/benchmarks/reference/methodology.md index 130b5c2..02486d8 100644 --- a/docs/benchmarks/reference/methodology.md +++ b/docs/benchmarks/reference/methodology.md @@ -2,7 +2,7 @@ The stemmer comparison suite measures Radixor and Java stemmers on the same language and deterministic Radixor model dictionary-derived data. Published Radixor rows in this refresh use contracted compiled patch tries, where uniform preferred-command subtrees are collapsed into accepting leaves before the trie is frozen for lookup. For each language, the registered default model resource stores the expected root as the first tab-separated field on a line and its surface forms on the same line. Every single-token field on that line can therefore be paired with the same expected root. -Published speed results come only from the exact method selection retained in `published-speed-benchmarks-2026-07-23.txt`. Internal `FrequencyTrie*` microbenchmarks, quality methods, the CISTEM gold-standard experiment, and the optional `PolishPolimorfStemmerComparisonBenchmark` are not part of those results. +Published speed results come only from the exact method selection retained in `published-speed-benchmarks-2026-08-10.txt`. Internal `FrequencyTrie*` microbenchmarks, quality methods, the CISTEM gold-standard experiment, and the optional `PolishPolimorfStemmerComparisonBenchmark` are not part of those results. The Snowball 3.1.0 refresh adds direct Czech, Persian, and Polish workloads; the existing Radixor and Lucene workload domains are unchanged. ## Benchmark Passes @@ -41,8 +41,8 @@ For right-to-left Radixor languages, patch application uses the traversal direct The quality pass reports exact-root agreement against the expected root from the default-model dictionary line. External-stemmer counters are written to: -- `build/reports/jmh/stemmer-accuracy-2026-07-23.csv` -- `build/reports/jmh/stemmer-accuracy-2026-07-23.txt` +- `build/reports/jmh/stemmer-accuracy-2026-08-10.csv` +- `build/reports/jmh/stemmer-accuracy-2026-08-10.txt` Accuracy is computed from standard JMH secondary rows: diff --git a/docs/benchmarks/reference/reproducibility.md b/docs/benchmarks/reference/reproducibility.md index 2571e48..e1b4d8e 100644 --- a/docs/benchmarks/reference/reproducibility.md +++ b/docs/benchmarks/reference/reproducibility.md @@ -4,19 +4,28 @@ - Machine-readable CSV: [stemming-quality.csv](../data/stemming-quality.csv) - SHA-256 record: [stemming-quality.sha256](../data/stemming-quality.sha256) -- SHA-256: `edf16b07be8a535943ddf37caeb8807755c95e9e1fb13244145f28be74b491d8` -- Complete scenarios: 308 +- SHA-256: `d34f325da320a2e040b54d8d8b5c216d70448f08cfb8659a423e99882aa1afb5` +- Complete scenarios: 314 - Authoritative language universe: 20 languages -- Language-page scenarios: 308 across 20 benchmark pages +- Language-page scenarios: 314 across 20 benchmark pages The CSV contains the model ID, independent model version, descriptor SHA-256, raw pair counts, raw over/under numerators and denominators, candidate statistics, and relation metrics. Reserved partition-metric columns remain empty because the gold standard is an overlapping cover. Documentation is regenerated from this file rather than manually transcribed. Publication fails when any row uses a model other than the language's registered default. ## Commands ```bash -./gradlew stemmingQuality -./gradlew publishStemmingQualityDocumentation -./gradlew verifyStemmingQualityDocumentation +./gradlew --no-daemon stemmingQuality \ + publishStemmingQualityDocumentation \ + verifyStemmingQualityDocumentation +./gradlew --no-daemon benchmarkCorpusReport writeJmhRuntimeClasspath +tools/run-published-accuracy-benchmarks.sh 2026-08-10 +tools/run-published-speed-benchmarks.sh 2026-08-10 +python3 tools/update-benchmark-documentation.py \ + --corpus build/reports/jmh/benchmark-corpora.csv \ + --accuracy build/reports/jmh/stemmer-accuracy-2026-08-10.csv \ + --speed build/reports/jmh/stemmer-speed-2026-08-10.csv \ + --coverage-accuracy build/reports/jmh/english-coverage-accuracy-2026-08-10.csv \ + --coverage-speed build/reports/jmh/english-coverage-speed-2026-08-10.csv ./gradlew test ./gradlew prepareMkDocsSource mkdocs build --strict --config-file build/mkdocs/mkdocs.yml @@ -59,18 +68,19 @@ The Pages workflow publishes that staged documentation together with Javadoc, JU ## Performance benchmark reproduction -The current speed and coverage-speed command is: +The current accuracy, speed, and coverage commands are: ```bash -./gradlew writeJmhRuntimeClasspath --no-daemon -tools/run-published-speed-benchmarks.sh 2026-07-23 +./gradlew --no-daemon benchmarkCorpusReport writeJmhRuntimeClasspath +tools/run-published-accuracy-benchmarks.sh 2026-08-10 +tools/run-published-speed-benchmarks.sh 2026-08-10 ``` -The runner refuses to start unless every CPU uses the `performance` governor, materializes the exact selected benchmark list, rejects quality/Polimorf/gold-standard methods, and requires the Hebrew speed path. It records hardware, JVM, source-state, JAR, classpath, corpus, quality, load, temperature, and governor provenance before running. The exact JMH configuration is listed in [Environment and reports](environment.md). Quality and performance reports are separate datasets and are not combined into an undocumented scalar. +The speed runner refuses to start unless every CPU uses the `performance` governor, materializes the exact selected benchmark list, rejects quality/Polimorf/gold-standard methods, and requires the Hebrew speed path. It records hardware, JVM, source-state, JAR, classpath, corpus, quality, load, temperature, and governor provenance before running. The accuracy runner evaluates all four exact-root benchmark classes and verifies that every new Snowball 3.1.0 candidate exposes all six accuracy counters. The exact JMH configuration is listed in [Environment and reports](environment.md). Quality and performance reports are separate datasets and are not combined into an undocumented scalar. ## Recorded and unavailable provenance -The performance documentation records its 2026-07-23 environment, JDK 25.0.3, operating system, hardware, base revision, exact dirty patch, untracked-source checksums, executable JMH JAR checksum, and model descriptor checksums. The quality CSV embeds model identity and checksum in every row; run date, core source state, JVM, OS, and hardware are shared provenance on the environment page. +The performance documentation records its 2026-08-10 environment, JDK, operating system, hardware, base revision, exact dirty patch, untracked-source checksums, executable JMH JAR checksum, and model descriptor checksums. The quality CSV embeds model identity and checksum in every row; run date, core source state, JVM, OS, and hardware are shared provenance on the environment page. Exact immutable upstream revisions were not recorded for every legacy UniMorph import. That limitation remains explicit in model descriptors and cannot be repaired from filesystem timestamps. Dependency versions reproducible from repository configuration include Apache Lucene 10.5.0, Morfologik 2.1.9, the Ukrainian dictionary artifact 4.9.1, and JMH 1.37. diff --git a/docs/benchmarks/reference/tested-stemmers.md b/docs/benchmarks/reference/tested-stemmers.md index 285f6fe..90100e6 100644 --- a/docs/benchmarks/reference/tested-stemmers.md +++ b/docs/benchmarks/reference/tested-stemmers.md @@ -4,10 +4,10 @@ The JMH adapter registry is authoritative for evaluated implementations and lang | Family or implementation | Upstream / attribution | Tested version or revision | Evaluated scope | Output capability and adapter behaviour | Interpretation notes | | --- | --- | --- | --- | --- | --- | -| Radixor | Egothor / Radixor project | Base commit and measured working-tree state recorded on the environment page | All 20 reconciled default model languages; all 20 have benchmark pages | Deterministic preferred patch via `get`; ranked distinct alternatives via `getAll`; primary is always included | Model-dictionary-derived compiled patch trie. Default rows use each language's stable default model ID. | +| Radixor | Egothor / Radixor project | Base commit and measured working-tree state recorded on the environment page | All 20 reconciled default model languages; all 20 have benchmark pages | Deterministic preferred patch via `get`; ranked distinct alternatives via `getAll`; primary is always included | Dictionary-trained compiled patch trie. Default rows use each language's stable default model ID. | | Apache Lucene language stem filters | Apache Lucene project | 10.5.0 | Adapter-declared language-specific subsets | TokenFilter lifecycle and language normalization match JMH; normally single-output | Light, minimal, possessive, and language stem filters deliberately implement different scopes. Narrow scope is not a defect. | | Apache Lucene SnowballFilter | Apache Lucene project using Snowball algorithms | Lucene 10.5.0 | Snowball-supported subset of Radixor languages | Single primary token emitted through the Lucene TokenFilter path | Includes TokenStream overhead and required normalization. | -| Official Snowball Java | Snowball project | Repository preparation downloads the configured upstream Java distribution; an immutable revision was not recorded in the quality CSV | Same-language adapter subset | Direct generated Java API; single output | Rule-based suffix algorithms provide broad baselines rather than dictionary-root guarantees. | +| Official Snowball Java | Snowball project | 3.1.0 source distribution; SHA-256 `5dab34d491f55f47b6e971569ffe6aadf5991512c648ddfe5d331b494cf6d655` | 17 same-language direct adapters, including the Czech, Persian, and Polish stemmers added in 3.1.0 | Direct generated Java API; single output | Rule-based suffix algorithms provide broad baselines rather than dictionary-root guarantees. Lucene 10.5.0 does not yet expose the three new algorithms through `SnowballFilter`, so those rows are direct-only. | | Lucene Stempel | Apache Lucene / Polish stemming tables | Lucene 10.5.0 | Polish | Direct and TokenFilter paths where registered; single primary output | Table-driven Polish implementation. | | Morfologik | Morfologik project; Lucene integration by Apache Lucene | Morfologik 2.1.9, Lucene integration 10.5.0; Ukrainian dictionary artifact 4.9.1 | Registered Polish and Ukrainian paths | Deterministic first lemma for primary comparison; all distinct lemma strings for candidate policies | Several analyses may share a lemma and are deduplicated by exact string equality. | | Hunspell via Lucene | Hunspell dictionaries from the `wooorm/dictionaries` repository; adapter by Apache Lucene | Lucene 10.5.0; dictionary repository revision was not recorded | Configured German, English, Spanish, French, Dutch, Polish, and Ukrainian dictionaries | First emitted stem is primary; all distinct stems at the token position are candidates | Dictionary content and affix rules differ by language. | diff --git a/docs/built-in-languages.md b/docs/built-in-languages.md index 3d6b261..40fa805 100644 --- a/docs/built-in-languages.md +++ b/docs/built-in-languages.md @@ -1,33 +1,42 @@ # Built-in Languages and Default Models -“Supported language” means that Radixor defines a language enum value and publishes a corresponding default model artifact. It does not mean that a dictionary is embedded in the core JAR. Applications add model artifacts explicitly or use the optional standard pack. +“Supported language” means that the repository maintains a default dictionary +and runtime mapping for that language. Packaging differs by runtime: Java keeps +the core dictionary-free and resolves external model artifacts, while the +Python installs a separate `radixor-models-standard` data package containing +the 20 default dictionaries in precompiled version 7 form. -The language enum carries language identity, writing direction, a legacy resource-directory name, and the stable default model ID. A model descriptor carries the independently versioned model identity and resource. See [Model Selection and Loading](model-selection-and-loading.md) for the API and the generated [model catalog](stemmer-model-catalog.md) for versions, provenance, checksums, and sizes. +The Java language enum carries language identity, writing direction, a legacy +resource-directory name, and the stable default model ID. A Java model +descriptor carries the independently versioned model identity and resource. +Python accepts the short alias or the same full model ID. See +[Model Selection and Loading](model-selection-and-loading.md) for Java and +[Python Usage and API](python/usage.md) for Python. ## Defaults and variants -| Language | Enum | Default model ID | Default artifact | Optional variants | -|---|---|---|---|---| -| Czech | `CS_CZ` | `cs-cz-default` | `org.egothor:radixor-model-cs-cz-default` | — | -| Danish | `DA_DK` | `da-dk-default` | `org.egothor:radixor-model-da-dk-default` | — | -| German | `DE_DE` | `de-de-default` | `org.egothor:radixor-model-de-de-default` | — | -| Spanish | `ES_ES` | `es-es-default` | `org.egothor:radixor-model-es-es-default` | — | -| Persian | `FA_IR` | `fa-ir-default` | `org.egothor:radixor-model-fa-ir-default` | — | -| Finnish | `FI_FI` | `fi-fi-default` | `org.egothor:radixor-model-fi-fi-default` | — | -| French | `FR_FR` | `fr-fr-default` | `org.egothor:radixor-model-fr-fr-default` | — | -| Hebrew | `HE_IL` | `he-il-default` | `org.egothor:radixor-model-he-il-default` | — | -| Hungarian | `HU_HU` | `hu-hu-default` | `org.egothor:radixor-model-hu-hu-default` | — | -| Italian | `IT_IT` | `it-it-default` | `org.egothor:radixor-model-it-it-default` | — | -| Norwegian Bokmål | `NB_NO` | `nb-no-default` | `org.egothor:radixor-model-nb-no-default` | — | -| Dutch | `NL_NL` | `nl-nl-default` | `org.egothor:radixor-model-nl-nl-default` | — | -| Norwegian Nynorsk | `NN_NO` | `nn-no-default` | `org.egothor:radixor-model-nn-no-default` | — | -| Polish | `PL_PL` | `pl-pl-unimorph` | `org.egothor:radixor-model-pl-pl-unimorph` | `pl-pl-polimorf` / `org.egothor:radixor-model-pl-pl-polimorf` | -| Portuguese | `PT_PT` | `pt-pt-default` | `org.egothor:radixor-model-pt-pt-default` | — | -| Russian | `RU_RU` | `ru-ru-default` | `org.egothor:radixor-model-ru-ru-default` | — | -| Swedish | `SV_SE` | `sv-se-default` | `org.egothor:radixor-model-sv-se-default` | — | -| Ukrainian | `UK_UA` | `uk-ua-default` | `org.egothor:radixor-model-uk-ua-default` | — | -| English | `US_UK` | `us-uk-default` | `org.egothor:radixor-model-us-uk-default` | — | -| Yiddish | `YI` | `yi-default` | `org.egothor:radixor-model-yi-default` | — | +| Language | Java enum | Python alias | Default model ID | Java default artifact | Optional variants | +|---|---|---|---|---|---| +| Czech | `CS_CZ` | `cs` | `cs-cz-default` | `org.egothor:radixor-model-cs-cz-default` | — | +| Danish | `DA_DK` | `da` | `da-dk-default` | `org.egothor:radixor-model-da-dk-default` | — | +| German | `DE_DE` | `de` | `de-de-default` | `org.egothor:radixor-model-de-de-default` | — | +| Spanish | `ES_ES` | `es` | `es-es-default` | `org.egothor:radixor-model-es-es-default` | — | +| Persian | `FA_IR` | `fa` | `fa-ir-default` | `org.egothor:radixor-model-fa-ir-default` | — | +| Finnish | `FI_FI` | `fi` | `fi-fi-default` | `org.egothor:radixor-model-fi-fi-default` | — | +| French | `FR_FR` | `fr` | `fr-fr-default` | `org.egothor:radixor-model-fr-fr-default` | — | +| Hebrew | `HE_IL` | `he` | `he-il-default` | `org.egothor:radixor-model-he-il-default` | — | +| Hungarian | `HU_HU` | `hu` | `hu-hu-default` | `org.egothor:radixor-model-hu-hu-default` | — | +| Italian | `IT_IT` | `it` | `it-it-default` | `org.egothor:radixor-model-it-it-default` | — | +| Norwegian Bokmål | `NB_NO` | `nb` | `nb-no-default` | `org.egothor:radixor-model-nb-no-default` | — | +| Dutch | `NL_NL` | `nl` | `nl-nl-default` | `org.egothor:radixor-model-nl-nl-default` | — | +| Norwegian Nynorsk | `NN_NO` | `nn` | `nn-no-default` | `org.egothor:radixor-model-nn-no-default` | — | +| Polish | `PL_PL` | `pl` | `pl-pl-unimorph` | `org.egothor:radixor-model-pl-pl-unimorph` | `pl-pl-polimorf` / `org.egothor:radixor-model-pl-pl-polimorf` | +| Portuguese | `PT_PT` | `pt` | `pt-pt-default` | `org.egothor:radixor-model-pt-pt-default` | — | +| Russian | `RU_RU` | `ru` | `ru-ru-default` | `org.egothor:radixor-model-ru-ru-default` | — | +| Swedish | `SV_SE` | `sv` | `sv-se-default` | `org.egothor:radixor-model-sv-se-default` | — | +| Ukrainian | `UK_UA` | `uk` | `uk-ua-default` | `org.egothor:radixor-model-uk-ua-default` | — | +| English | `US_UK` | `en` | `us-uk-default` | `org.egothor:radixor-model-us-uk-default` | — | +| Yiddish | `YI` | `yi` | `yi-default` | `org.egothor:radixor-model-yi-default` | — | The maintained table deliberately avoids duplicating mutable provenance and checksum fields. Those values come from module metadata and are generated into the model catalog. @@ -44,6 +53,11 @@ The maintained table deliberately avoids duplicating mutable provenance and chec UniMorph and PoliMorf have different lexical sources and provenance. Applications should compare outputs with application-specific regression tests before changing an explicit model choice. +In Python, `Stemmer("pl")` selects `pl-pl-unimorph`. The standard Python data +package does not include PoliMorf; applications that need it must compile and +load it explicitly as a trusted custom model. As in Java, it never changes the +Polish default implicitly. + ## Dependency patterns Minimal English: diff --git a/docs/cli-compilation.md b/docs/cli-compilation.md index 29c7ab0..a6800f9 100644 --- a/docs/cli-compilation.md +++ b/docs/cli-compilation.md @@ -1,7 +1,11 @@ -# CLI Compilation +# Java CLI Compilation Radixor provides a command-line compiler for turning line-oriented dictionary files into compact binary stemmer artifacts. +This page documents the Java CLI and its selectable reduction and normalization +controls. Python exposes its production compilation profile through +`radixor.compile(...)`; see [Compiling Dictionaries in Python](python/model-compilation.md). + The CLI output is not a model JAR. A model artifact contains a compressed textual dictionary, descriptor, index, checksum, and license so the runtime registry can discover and compile it. The CLI instead emits an already compiled binary trie for direct `loadBinaryCompiled(...)` use. Choose the model-module workflow when independently published classpath discovery is required; choose the CLI when the application owns a compiled binary asset. This is the preferred preparation workflow when stemming should run against an already compiled artifact rather than against raw dictionary input. The CLI reads the dictionary, derives patch commands, builds a mutable trie, applies the selected subtree reduction strategy, and writes the final compiled trie in the project binary format under GZip compression. The result is a deployment-ready `.radixor.gz` file that can be loaded directly by application code. diff --git a/docs/dictionary-format.md b/docs/dictionary-format.md index af707b8..a3cec9b 100644 --- a/docs/dictionary-format.md +++ b/docs/dictionary-format.md @@ -8,11 +8,16 @@ Three artifacts must not be confused: | Artifact | Representation | Consumer | |---|---|---| -| Source textual dictionary | Plain UTF-8 tab-separated rows | Authors, parser, CLI, or model preparation | -| Registered model resource | The same Radixor dictionary bytes under GZip, accompanied by index, descriptor, checksum, and license | `StemmerModelRegistry` and `StemmerPatchTrieLoader` | -| Persisted compiled trie | GZip-compressed Radixor binary format, commonly `.radixor.gz` | `loadBinaryCompiled(...)` | +| Source textual dictionary | Plain UTF-8 tab-separated rows, optionally GZip-compressed | Java and Python loaders or compilation tools | +| Registered Java model resource | The same dictionary bytes under GZip, accompanied by index, descriptor, checksum, and license | Java `StemmerModelRegistry` and `StemmerPatchTrieLoader` | +| Standard Python model resource | A precompiled version 7 `.rxc` artifact in the required `radixor-models-standard` data package | Python `Stemmer("")` | +| Persisted compiled trie | GZip-compressed Radixor version 7 binary, commonly `.radixor.gz` in Java or `.rxc` in Python | Java `loadBinaryCompiled(...)` and Python `Stemmer(compiled=...)` | -The model file named `stemmer.gz` is not Java serialization and is not a pre-instantiated or persisted trie. It is compressed textual dictionary input parsed when the model is loaded. +The Java model file named `stemmer.gz` is not Java serialization and is not a +pre-instantiated or persisted trie. It is compressed textual dictionary input +parsed when a Java model is loaded. Python's standard data distribution instead +contains generated, precompiled version 7 tries; neither Python distribution +ships the textual source dictionaries. Consequently, compressed size is not a construction-memory estimate. The PoliMorf resource is 12,624,997 bytes compressed and 68,093,680 bytes decompressed, while full parsing, trie construction, reduction, and patch compilation require a dedicated verification JVM with a 6 GiB maximum heap. @@ -25,6 +30,22 @@ generation are disclosed transformations; the in-memory trie is a Radixor runtim Each logical line describes one canonical stem and zero or more known word variants that should reduce to that stem. The format is intentionally lightweight, easy to maintain in source control, and directly consumable both by the programmatic loader and by the CLI compiler. +## Use the format from either runtime + +The dictionary semantics are shared; the integration entry points are not: + +| Task | Java | Python | +|---|---|---| +| Load and compile text now | `StemmerPatchTrieLoader.loadCompiled(...)` | `Stemmer(path=...)` | +| Compile a reusable binary | `org.egothor.stemmer.Compile` | `radixor.compile(...)` | +| Load a compiled binary | `StemmerPatchTrieLoader.loadBinaryCompiled(...)` | `Stemmer(compiled=...)` | +| Registered/standard language | External model JAR selected through the registry | Compiled model in `radixor-models-standard`, selected by alias or model ID | + +Python compilation writes the shared version 7 stream using its fixed +production reduction profile. Java exposes additional reduction and +normalization controls. See [Java CLI Compilation](cli-compilation.md) and +[Compiling Dictionaries in Python](python/model-compilation.md). + ## Core structure Each non-empty logical line has the following shape: diff --git a/docs/fast-track.md b/docs/fast-track.md index 49a1d87..2d66283 100644 --- a/docs/fast-track.md +++ b/docs/fast-track.md @@ -1,10 +1,13 @@ -# Fast Track +# Java Fast Track This page is the shortest path from an empty Java project to a working Radixor stemmer. It deliberately uses an external model artifact and the preferred compiled-command runtime API, so the first result does not require writing a dictionary, running the CLI compiler, or understanding reduction internals. +For the native Python package, start with the [Python Fast Track](python/fast-track.md) +instead; Python does not use Maven model artifacts or the Java loader API. + Use this page when the goal is: - add the dependency, diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..d2e604a --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,34 @@ +# Get Started with Radixor + +Radixor provides first-class Java and Python runtimes over the same learned +patch-command model. Choose the environment that matches the application; both +use the maintained language catalog and can exchange compiled version 7 tries. + +## Python + +Choose Python for native CPython integration, efficient batch calls, and a +PyStemmer-compatible migration surface. + +```bash +python -m pip install radixor +``` + +- [Python Fast Track](python/fast-track.md) — install and produce the first stem. +- [Python Quick Start](python/quick-start.md) — build an application workflow. +- [Python Usage and API](python/usage.md) — scalar, batch, cache, and custom-model APIs. + +## Java + +Choose Java for JVM integration, selectable reduction modes, registry-based +model discovery, and extending or persisting compiled tries. + +- [Java Fast Track](fast-track.md) — add the core/model dependencies and produce the first stem. +- [Java Quick Start](quick-start.md) — loading, querying, extension, and persistence. +- [Java Integration Deep Dive](integration-deep-dive.md) — lifecycle and production guidance. + +## Shared concepts + +- [Built-in Languages](built-in-languages.md) documents aliases, model IDs, and defaults. +- [Dictionary Format](dictionary-format.md) defines the common source semantics. +- [Benchmarks](benchmarks/index.md) separates runtime speed from linguistic quality. +- [Architecture](architecture.md) explains shared behavior and runtime-specific internals. diff --git a/docs/index.md b/docs/index.md index dd7d4e9..2d0caa0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,53 +1,9 @@ -

Home

-

- Radixor banner -

+--- +template: landing.html +title: Radixor +description: Learned transformation stemming for Java and Python using compact reduced tries of patch commands across 20 languages. +--- -**Radixor** is a high-performance, multi-language stemmer for Java, built for production-grade search and text-processing systems. +# Radixor -It modernizes the proven Egothor patch-command trie approach and extends it for deployment realities that classic stemming pipelines do not handle well. - -Traditional Egothor-style stemming workflows usually treat a compiled dictionary as a fixed artifact. Once built, its lexical knowledge is effectively closed unless the original source dictionary is recompiled. Radixor removes that constraint. An already compiled stemming structure can be extended with additional words and transformations, which makes it possible to evolve an existing dictionary for domain-specific, customer-specific, or deployment-specific vocabulary without rebuilding the entire lexical base from scratch. - -Radixor also improves how ambiguous reductions can be handled at runtime. Instead of always forcing a single result, it can return multiple plausible stems when the input token cannot be reduced unambiguously. This allows downstream systems to preserve linguistic ambiguity where that is operationally useful, whether for retrieval quality, ranking strategies, diagnostics, or domain-specific normalization policies. - -The project also has a clear research lineage. The historical idea behind this stemming family is described in Leo Galambos's paper *Lemmatizer for Document Information Retrieval Systems in JAVA* (SOFSEM 2001), which presents a semi-automatic stemming technique designed for Java-based information retrieval systems. In Radixor documentation, this reference serves as historical and algorithmic background rather than as technical documentation of the current implementation. - -> Unlike traditional Egothor-based deployments, Radixor can extend an already compiled stemmer dictionary and can return multiple stems when a word is not reducible to a single unambiguous form. - -Radixor delivers: - -- **Fast runtime stemming** with compact lookup structures -- **Multi-language adaptability** through dictionary-driven compilation -- **Extension of compiled stemmer structures** without full recompilation from source dictionaries -- **Incremental vocabulary growth** for deployment-specific lexical refinement -- **Support for multiple stemming results** when reduction is ambiguous -- **Deterministic behavior** suitable for reproducible processing pipelines -- **Flexible integration paths**, including CLI-based and programmatic workflows -- **Operational transparency** through continuously published quality and benchmark reports - -Radixor is intended for teams that require consistent stemming quality at scale, while retaining the ability to evolve lexical resources after compilation and to handle ambiguous reductions with greater precision than traditional single-stem pipelines allow. - -## Add the core and model data - -The core `org.egothor:radixor` JAR contains no language dictionary. A minimal application adds one model; broad deployments may use the optional standard pack: - -```groovy -dependencies { - implementation 'org.egothor:radixor:' - runtimeOnly 'org.egothor:radixor-model-pl-pl-unimorph:1.0.0' -} -``` - -`StemmerPatchTrieLoader.loadCompiled(Language.PL_PL, ...)` resolves the default `pl-pl-unimorph`. `pl-pl-polimorf` is a separate optional model selected by stable model ID. Follow [Model Selection and Loading](model-selection-and-loading.md) for runnable examples or choose artifacts from the generated [model catalog](stemmer-model-catalog.md). - -## Start here - -- Read [Fast Track](fast-track.md) when you want the shortest path to a working bundled stemmer. -- Use [Model Selection and Loading](model-selection-and-loading.md) for default, explicit, dual-model, and ClassLoader examples. -- Use [Integration Deep Dive](integration-deep-dive.md) when you are wiring Radixor into a real application or search pipeline. -- Read [Quick Start](quick-start.md) for the broader developer walkthrough after the first result works. -- Use [Built-in Languages](built-in-languages.md) to interpret language defaults and optional model variants. -- Review [Benchmarking](benchmarking.md) and [Benchmark Results](benchmarks/index.md) for reproducible performance and quality methodology. -- Open [CI Reports](reports.md) to inspect published build artifacts and quality metrics. -- See the historical paper: [*Lemmatizer for Document Information Retrieval Systems in JAVA*](https://www.researchgate.net/publication/221512865_Lemmatizer_for_Document_Information_Retrieval_Systems_in_JAVA). +Learned transformation stemming for Java and Python, using compact reduced tries of patch commands and reproducible quality/performance benchmarks across 20 languages. diff --git a/docs/integration-deep-dive.md b/docs/integration-deep-dive.md index 69efb5f..6abfc7d 100644 --- a/docs/integration-deep-dive.md +++ b/docs/integration-deep-dive.md @@ -1,4 +1,4 @@ -# Integration Deep Dive +# Java Integration Deep Dive This page explains how to integrate Radixor into a real Java application after the first fast-track experiment works. It covers dependencies, external model artifacts, runtime lifecycle, diff --git a/docs/overrides/landing.html b/docs/overrides/landing.html new file mode 100644 index 0000000..37ff756 --- /dev/null +++ b/docs/overrides/landing.html @@ -0,0 +1,133 @@ +{% extends "base.html" %} +{% block extrahead %}{{ super() }}{% endblock %} +{% block header %}
RADIXOR
{% endblock %} +{% block tabs %}{% endblock %} +{% block site_nav %}{% endblock %} +{% block container %} +
+
+
+
+
Dictionary-trained. Not dictionary-bound.
+

High-quality multilingual stemming.
Built for Java and Python.

+

Radixor learns word-to-stem transformations from lexical evidence, compiles them into a reduced trie of patch commands, and applies those commands algorithmically at runtime. The training dictionary supplies evidence—it does not define a closed runtime vocabulary.

+
+ +
+
Choose your runtime
+
+ ♨Get Radixor for Java → + PyGet Radixor for Python → +
+
+
Learned transformationsTrie + compact patch commands
+
GeneralizesNot restricted to listed word forms
+
ReproduciblePublic quality and speed evidence
+
+
+ Why Radixor is different → + See the benchmarks → + Choose a documentation path → +
+
+
+ +
+ +
+
+ +
+
+ Java benchmark award +

Java: wins in all 20
benchmarked languages

In the current same-language Java matrix, Radixor records the lowest measured runtime against every configured comparator on all 20 language pages.

+
20 / 20Benchmark winsAcross all 20 languages
+
Reproducible · Transparent · Verifiable

Every result is public, verifiable, and reproducible.

View all benchmarks →
+
+ +

Quality and speed you can trust — Finnish case study

+
+
Radixor❄ Snowball Finnish Radixor advantage
+
Quality score(higher is better)0.9848~0.740+33% higher quality
+
◌Overstemming(lower is better)3.03%~52%94.2% lower
+
Speed(vs. PyStemmer)1.14× faster1.00× (baseline)1.14× faster
+
+

Quality comparator: published Snowball Finnish Lucene/Snowball results. Speed comparator: PyStemmer 3.1.0 at batch size N=100. The point is the combined quality/performance envelope, not wrapper identity.

+ +
+
20 languagesFull Unicode support
+
Czech flagCzechDanish flagDanishDutch flagDutchEnglish flagEnglishFinnish flagFinnishFrench flagFrenchGerman flagGermanHebrew flagHebrewHungarian flagHungarianItalian flagItalianNorwegian Bokmål flagNorwegian BokmålNorwegian Nynorsk flagNorwegian NynorskPersian flagPersianPolish flagPolishPortuguese flagPortugueseRussian flagRussianSpanish flagSpanishSwedish flagSwedishUkrainian flagUkrainianYiddish flagYiddish
+
+ +
+
♨Java
import org.egothor.stemmer.StemmerPatchTrieLoader;
+import org.egothor.stemmer.FrequencyTrie;
+
+FrequencyTrie<CompiledPatchCommand> trie =
+    StemmerPatchTrieLoader.loadCompiled(
+        StemmerPatchTrieLoader.Language.US_UK, ...);
+
+String word = "running";
+CompiledPatchCommand patch = trie.get(word);
+String stem = patch == null ? word : patch.apply(word);
View Java docs →
+
PyPython
from radixor import Stemmer
+
+stemmer = Stemmer("en")
+stems = stemmer.stem_batch([
+    "running", "studies", "better", "cars"
+    ])
View Python docs →
+
+ +
+

Learned transformationsLexical evidence becomes
compact patch commands

+

Beyond the dictionaryRuntime is not a closed
word-to-lemma lookup

+

Native speedMulti-million words per
second in Python and Java

+

20 language modelsOne architecture, trained
from language-specific data

+

Production readyDeterministic behavior
and reproducible evidence

+
+ +
+ +

A different stemming architecture.

Radixor combines learned patch commands, trie-based structural sharing, subtree reduction, and deterministic runtime application. It is neither a flat dictionary lookup nor another fixed suffix-rule table.

+ Technology & lineage + ▣  Read the Docs +
+
+
+{% endblock %} +{% block footer %}{% endblock %} diff --git a/docs/programmatic-loading-and-building.md b/docs/programmatic-loading-and-building.md index e500479..5c57ef9 100644 --- a/docs/programmatic-loading-and-building.md +++ b/docs/programmatic-loading-and-building.md @@ -1,7 +1,10 @@ -# Loading and Building Stemmers +# Loading and Building Stemmers in Java This document explains how to acquire a compiled Radixor stemmer in Java. +For Python construction and binary preparation, use [Python Usage and API](python/usage.md) +and [Compiling Dictionaries in Python](python/model-compilation.md). + ## Load a registered default model Language-oriented entry points resolve a registered default model and compile its GZip textual dictionary into a `FrequencyTrie`. The corresponding model JAR must be on the runtime classpath; the core contains no dictionary. diff --git a/docs/programmatic-usage.md b/docs/programmatic-usage.md index f2c23ae..d87d620 100644 --- a/docs/programmatic-usage.md +++ b/docs/programmatic-usage.md @@ -1,7 +1,12 @@ -# Programmatic Usage +# Java Programmatic Usage Radixor code and model data are separate runtime components. Every example on this page requires `org.egothor:radixor:` as an `implementation` dependency and at least one model JAR as a runtime dependency. The core JAR contains no `stemmer.gz`. +The Python implementation has its own native API. `pip install radixor` also +installs the separate standard data package containing 20 precompiled models. +See the [Python Quick Start](python/quick-start.md) and +[Python Usage and API](python/usage.md). + For complete dependency patterns, lifecycle guidance, and troubleshooting, use [Model Selection and Loading](model-selection-and-loading.md). The generated [model catalog](stemmer-model-catalog.md) records the current artifacts, versions, checksums, and provenance. ## 1. Minimal use: the Polish default diff --git a/docs/python/fast-track.md b/docs/python/fast-track.md new file mode 100644 index 0000000..ce46ea7 --- /dev/null +++ b/docs/python/fast-track.md @@ -0,0 +1,66 @@ +# Python Fast Track + +This is the shortest path from an empty Python environment to a working +Radixor stemmer. The installation includes the native runtime and the separate +standard-model package with 20 precompiled language models. + +## 1. Install + +=== "PyPI" + + ```bash + python -m pip install --only-binary=:all: radixor + ``` + +=== "GitHub Releases" + + ```bash + python -m pip install --only-binary=:all: \ + --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor + ``` + +PyPI publication is pending, and the GitHub index becomes live with the first +Python releases. Until then, follow the source-checkout procedure on +[Installation and Builds](installation.md). + +Radixor supports CPython 3.9 and newer. A JVM, Java dependency, and source +dictionary are not required. + +## 2. Stem words + +```python +from radixor import Stemmer + +stemmer = Stemmer("en") + +print(stemmer.stem("running")) +print(stemmer.stem_batch(["running", "studies", "cars"])) +``` + +Expected first output: + +```text +run +``` + +`stem()` and `stem_batch()` preserve Radixor's original API: a word for which +the trie finds no patch command produces `None`. + +## 3. Use PyStemmer-compatible fallback semantics + +For a low-friction migration from PyStemmer, use the compatible method names: + +```python +stemmer.stemWord("running") +stemmer.stemWords(["running", "unknown_word"]) +``` + +These methods return the original input whenever no patch command is found, so +their results are always strings rather than `None`. + +## Next + +- Continue with the [Python Quick Start](quick-start.md) for model selection, + batch processing, custom compiled models, and deployment guidance. +- Use [Python Usage and API](usage.md) as the method reference. +- Review the reproducible [Python performance results](performance.md). diff --git a/docs/python/index.md b/docs/python/index.md new file mode 100644 index 0000000..dfdb820 --- /dev/null +++ b/docs/python/index.md @@ -0,0 +1,96 @@ +# Radixor for Python + +The **`radixor`** package is Radixor's native Python implementation. It is not +a wrapper around the Java library and does not require a JVM: it is a +compiled extension (Rust, via [PyO3](https://pyo3.rs/) and +[maturin](https://www.maturin.rs/)) that loads precompiled patch-command tries +derived from the same canonical UniMorph data as the Java models. + +```python +from radixor import Stemmer + +s = Stemmer("en") +s.stem("running") # 'run' +s.stem_batch(["cats", "ran"]) # ['cat', 'run'] +``` + +- [Fast Track](fast-track.md) — install and produce the first stem. +- [Quick Start](quick-start.md) — the complete application-oriented learning path. +- [Installation and building](installation.md) — Linux, Windows, macOS. +- [Usage and examples](usage.md) — batch API, caching, and custom models. +- [Dictionary compilation](model-compilation.md) — prepare a version 7 binary + once and share it with Python or Java. +- [Performance](performance.md) — fair, reproducible comparisons vs PyStemmer, + snowballstemmer, NLTK Porter, and CISTEM. + +The language and model-ID mapping is shared with Java and maintained on the +[Built-in Languages](../built-in-languages.md) page. Installing `radixor` +also resolves the separate pure `radixor-models-standard` distribution containing +the 20 default compiled models; Java applications select independently +versioned model JARs. + +!!! note "Same models, same results, different runtime" + The standard Python models are compiled from the identical canonical + dictionaries with the identical production reduction configuration + (`MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS`, 75 % / 3×, + uniform-subtree contraction, `LOWERCASE_WITH_LOCALE_ROOT`, `AS_IS` + diacritics, `storeOriginal=true`). For a word present in a model, both + implementations return the same dominant stem. The compiled **binary format + is shared** (see below), so a model compiled by one side loads in the other. + +## Java vs. Python: read this first + +The two implementations solve the same problem but make different runtime +trade-offs. Mixing their mental models causes confusion, so the differences are +stated explicitly. **Neither is “better”** — they target different runtimes. + +| Aspect | Java (`org.egothor:radixor`) | Python (`radixor`) | +|---|---|---| +| Runtime | JVM library | Compiled extension (Rust/PyO3), no JVM | +| Distribution | Maven JAR + model JARs | `abi3` wheel (one wheel per OS/arch, Python ≥ 3.9) | +| Hot-path data structure | `CompiledNode` graph; routines operate on caller-owned **`char[]`** with zero-copy normalized lookups and visitor sinks (`EntrySink`) | Flat **CSR arrays** (no per-node objects); reused UTF‑16 scratch buffers | +| Result cache | **None** — `get()` is stateless and re-stems every call | **Bounded**, 10,000 entries by default (matching PyStemmer); `Stemmer(cache_size=0)` disables it | +| Batch API | Not a batch call; you loop and reuse `char[]`/visitors to avoid allocation | **`stem_batch()` / `stem_all_batch()`** — one Python↔Rust crossing amortized over the whole list | +| Reduction modes | All three modes selectable at compile time | Fixed to the production `DOMINANT` mode | +| Extending a compiled trie | **Supported** — add words/transformations to an already-compiled trie without recompiling | **Not exposed** — compile from a dictionary (or load a compiled binary) | +| Model resolution | `ServiceLoader` registry, descriptors, SHA‑256 integrity checks | Separate standard data package; catalog/format/SHA‑256 validation before synchronous native load | +| Normalization control | Case and diacritic modes fully configurable | `lowercase` toggle; diacritics `AS_IS` (models are built this way) | +| Binary format | `StemmerPatchTrieBinaryIO` v7 read/write (versioned, fingerprinted) | v7 read/write, **inner stream byte-identical** to Java; **v7 only** (no legacy v1–v6) | +| Multiple stems | `getAll(...)` | `stem_all()` / `stem_all_batch()` | + +### Runtime capabilities that differ + +To avoid surprises, these Java capabilities are **not** in the Python package: + +- **Extending / incrementally growing a compiled trie.** Python compiles from a + source dictionary (or loads a compiled binary); it does not add words to an + existing compiled trie at runtime. +- **Selectable reduction modes.** Only the production `DOMINANT` mode is used. +- **Pluggable provider discovery.** Python currently resolves one known + standard provider directly; entry-point plugins are not yet exposed. +- **Legacy binary versions.** Only stream version 7 is read/written. +- **Diacritic-removal modes** beyond `AS_IS` (the bundled models are `AS_IS`). + +### Python-specific capabilities + +- A **batch API** (`stem_batch`) that amortizes the Python↔native boundary — the + single most important call for throughput from Python. +- A **bounded result cache** (`cache_size=10_000` by default) for workloads with + repeated tokens. It is shared by `stem()`, `stemWord()`, `stem_batch()`, and + `stemWords()`; pass `cache_size=0` to disable it. The `stem_all*()` methods are + not cached. +- A `lowercase=False` mode to skip per-lookup lowercasing when the caller + guarantees already-lowercased input. + +## Interoperability + +The compiled binary is Radixor's **v7 trie stream**, and the Python runtime writes +the *inner stream byte-for-byte identically to the Java* +`StemmerPatchTrieBinaryIO`. Consequently: + +- a model compiled by **Java** (`org.egothor.stemmer.Compile` / + `StemmerPatchTrieBinaryIO.write`) loads in **Python**, and +- a model compiled by **Python** (`radixor.compile(...)`) loads in **Java**. + +(The outer gzip wrapper bytes differ between the two gzip implementations; this +is irrelevant — both sides decompress to the same v7 stream.) diff --git a/docs/python/installation.md b/docs/python/installation.md new file mode 100644 index 0000000..f6f13d2 --- /dev/null +++ b/docs/python/installation.md @@ -0,0 +1,215 @@ +# Installing and building (Linux, Windows, macOS) + +The Python package ships as an **`abi3` wheel** — a single binary per +OS/architecture that works on CPython ≥ 3.9 (including 3.14) through the stable +ABI. Most users just `pip install`; building from source is only needed for +development or unsupported platforms. + +## Install from PyPI + +PyPI is the intended primary index once the Radixor projects are approved and +published there: + +```bash +python -m pip install --only-binary=:all: radixor +``` + +PyPI publication is not live yet. Until the `radixor` and +`radixor-models-standard` project pages exist, this command cannot install the +project. + +## Install compiled packages from GitHub + +Python releases are published as immutable GitHub Release assets. A small +PEP 503 index on GitHub Pages exposes both packages to `pip`: + +```bash +python -m pip install --only-binary=:all: \ + --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor +``` + +The index links directly to checksummed wheel assets in GitHub Releases; Pages +does not duplicate the package files. It is not live until the first Python +model and native releases have been published. This was verified before the +initial release: the URL returned HTTP 404 and the repository contained no +Python Release assets. + +Do not configure the GitHub index as an `--extra-index-url`: `pip` does not +prioritize one index over another. Use it as the sole `--index-url`, as shown +above. The binary-only constraint also prevents an accidental source build +with an unprepared toolchain. + +Wheels are provided for Linux (`manylinux`), Windows, and macOS +(x86‑64 and Apple Silicon). A source distribution is also published; installing +it triggers a source build, which needs the toolchain described below. + +## Install or build from the GitHub source repository + +Building requires the **Rust toolchain**, a linker for the target platform, and +**maturin**. The crate and its dependencies contain no project C/C++ sources, +but the selected Rust target still needs its normal platform linker and SDK. + +```bash +git clone https://github.com/leogalambos/Radixor +cd Radixor +python -m venv python/.venv +# activate the venv (see per-OS note below) +pip install maturin build setuptools wheel pytest +./gradlew pythonBuildStandardModels +pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl +cd python +maturin develop --release # compile + install into the venv +pytest -q # run the test suite +``` + +For a reproducible application build, check out a release tag or exact commit +instead of a moving branch. Repository descriptors deliberately use the +non-release placeholder `0.0.0`; release workflows inject the version from the +Git tag into isolated staging trees. Consequently, source-checkout development +installs use `--no-deps` for the generated development model wheel, while +published packages carry normal release versions and dependency resolution +works automatically. + +The native distribution requires +`radixor-models-standard>=1.0,<2.0`; an installation of `radixor` +resolves it automatically. The local `--no-deps` command installs the generated +data wheel for development without contacting a package index. + +The installed package source (package index, environment, and `sys.path`) is +the model-provider trust boundary. Manifest SHA-256 checks detect accidental +corruption after installation; they do not authenticate a malicious provider. + +## Integrity and provenance + +Every GitHub Release contains `SHA256SUMS` for its wheel and source archives. +The release workflows also create GitHub artifact attestations for those +archives. After downloading a release, maintainers and users can verify it with: + +```bash +sha256sum --check SHA256SUMS +gh attestation verify radixor--.whl \ + --repo leogalambos/Radixor +``` + +Python packages do **not** reuse the OpenPGP key configured for Java/Maven +Central publications. Java's `SIGNING_KEY` and `SIGNING_PASSWORD` produce Maven +signatures; Python currently uses release checksums plus GitHub's +identity-bound build-provenance attestation. A future PyPI publication should +use PyPI Trusted Publishing and its supported attestations rather than copying +the Java signing mechanism. + +## Build through Gradle + +From the repository root, the supported build entry point creates the native +wheel/sdist and pure standard-model wheel/sdist: + +```bash +./gradlew pythonBuild +``` + +Artifacts are written below `build/python/dist/`; they are not installed into +the invoking interpreter. `./gradlew pythonVerifyDistributions` also validates +archive contents, dependency metadata, checksums, v7 headers, and a fresh +offline wheel-only installation. Platform convenience tasks are also available: + +```bash +./gradlew pythonBuildLinux +./gradlew pythonBuildWindows +./gradlew pythonBuildMacos +``` + +The task matching the current host delegates to `pythonBuild`. A non-host task +uses the default Rust target for that operating system and therefore succeeds +only when its Rust target, linker, and platform SDK are installed. Override a +default with `pythonLinuxTarget`, `pythonWindowsTarget`, or `pythonMacosTarget`. +For example: + +```bash +./gradlew pythonBuildWindows -PpythonWindowsTarget=x86_64-pc-windows-gnu +``` + +Use `-PpythonExecutable=/path/to/python` or +`-PmaturinExecutable=/path/to/maturin` when those tools are not on `PATH`. +These Gradle tasks are the repository integration; direct `maturin` commands +remain useful while developing inside `python/`. + +### Prerequisites per platform + +=== "Linux" + + ```bash + # Rust (rustup); most distros already ship Python 3.9+ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + python3 -m venv .venv && source .venv/bin/activate + pip install maturin pytest + ``` + Producing distributable `manylinux` wheels is easiest with + `maturin build --release` inside the official maturin/`manylinux` container. + +=== "macOS" + + ```bash + brew install rustup-init && rustup-init -y # or: curl https://sh.rustup.rs | sh + python3 -m venv .venv && source .venv/bin/activate + pip install maturin pytest + maturin develop --release + ``` + Both Apple Silicon (`aarch64-apple-darwin`) and Intel + (`x86_64-apple-darwin`) are supported; `maturin build --release --target + universal2-apple-darwin` produces a universal wheel. + +=== "Windows" + + ```powershell + winget install -e --id Rustlang.Rustup + py -m venv .venv + .\.venv\Scripts\Activate.ps1 + pip install maturin pytest + maturin develop --release + ``` + The self-contained GNU toolchain avoids needing Visual Studio Build Tools: + ```powershell + rustup toolchain install stable-x86_64-pc-windows-gnu + rustup default stable-x86_64-pc-windows-gnu + ``` + (The MSVC toolchain also works if you already have the C++ Build Tools.) + +### Python 3.14 (and newer than your PyO3 knows about) + +Because the extension targets the stable ABI, it links against interpreters +newer than the PyO3 version was released for. If a build against a very new +CPython refuses, set the forward-compatibility flag once in the build shell: + +=== "Linux / macOS" + + ```bash + export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 + maturin develop --release + ``` + +=== "Windows (PowerShell)" + + ```powershell + $env:PYO3_USE_ABI3_FORWARD_COMPATIBILITY = "1" + maturin develop --release + ``` + +## Verifying the build + +```bash +python -c "from radixor import Stemmer; print(Stemmer('en').stem('running'))" # -> run +pytest -q +``` + +## Notes and caveats + +- **Model packaging.** Neither runtime distribution contains textual + dictionaries. `radixor-models-standard` ships 20 compiled gzip v7 resources, + the checksum/provenance manifest, and CC BY-SA 3.0 notices; optional + `pl-pl-polimorf` is excluded. +- **Catalog compatibility.** Radixor 4.1 accepts model-distribution major 1 + (`>=1.0,<2.0`) carrying the independent 2026.1 catalog identity. Missing, + incompatible, or corrupt data produces an + actionable error before native loading. +- **Toolchain PATH.** After installing rustup, open a fresh shell (or ensure + `~/.cargo/bin` is on `PATH`) so `maturin` can find `cargo`/`rustc`. diff --git a/docs/python/model-compilation.md b/docs/python/model-compilation.md new file mode 100644 index 0000000..e21fa50 --- /dev/null +++ b/docs/python/model-compilation.md @@ -0,0 +1,105 @@ +# Compiling Dictionaries in Python + +The Python package can compile a textual Radixor dictionary into the shared +version 7 binary trie format. This moves dictionary parsing, patch-command +generation, trie construction, reduction, and serialization out of application +startup. + +Use this workflow when the application owns its model file. Standard language +aliases already load validated, precompiled `.rxc` resources from +`radixor-models-standard`; they do not parse or compile textual dictionaries +when a `Stemmer` is constructed. + +## Source format + +The input is a plain UTF-8 or GZip-compressed UTF-8 tab-separated dictionary. +The first column is the canonical stem and the remaining columns are its known +surface forms: + +```text +run running runs ran +cat cats +``` + +Remarks beginning with `#` or `//` are accepted. The complete syntax and +normalization rules are documented on the shared [Dictionary Format](../dictionary-format.md) +page. + +## Compile a model + +```python +import radixor + +radixor.compile( + "stemmer.tsv.gz", + "english.rxc", + language="en", +) +``` + +`language` is used to choose traversal direction when `backward` is omitted. +Persian (`fa`), Hebrew (`he`), and Yiddish (`yi`) use forward traversal; the +other bundled languages use backward traversal. For a custom language, select +the direction explicitly: + +```python +radixor.compile( + "custom.tsv", + "custom.rxc", + backward=True, + store_original=True, + lowercase=True, +) +``` + +The arguments are: + +| Argument | Meaning | +|---|---| +| `source` | Plain or GZip-compressed textual dictionary. | +| `out_path` | Destination for the GZip-compressed version 7 trie. | +| `language` | Optional alias or model ID used only to infer traversal direction. | +| `backward` | Explicit traversal direction; overrides inference from `language`. | +| `store_original` | Include a no-op mapping for every canonical stem. Defaults to `True`. | +| `lowercase` | Record lowercase lookup normalization in the compiled metadata. Defaults to `True`. | + +Compilation refuses an input that is already a compiled trie. The destination +is written by the native extension; the caller is responsible for choosing its +location and for replacing an existing file only when that is intended. + +## Load the compiled model + +```python +from radixor import Stemmer + +stemmer = Stemmer(compiled="english.rxc") +print(stemmer.stem("running")) +``` + +`Stemmer(path=...)` also auto-detects textual dictionaries and compiled version +7 streams, but `compiled=` communicates the deployment intent more clearly. +Traversal direction, `store_original`, and lookup normalization are already +stored in a compiled artifact; constructor build options do not rewrite them. + +## Java interoperability + +Python and Java share the inner version 7 trie stream. A binary produced by +`radixor.compile(...)` can be loaded by Java's +`StemmerPatchTrieLoader.loadBinaryCompiled(...)`, and Python can load a version +7 artifact written by `StemmerPatchTrieBinaryIO`. + +The outer GZip bytes need not be identical because compressor implementations +may differ. Interoperability applies to the decompressed version 7 stream and +its persisted metadata. + +## Differences from the Java compiler + +Python compilation intentionally exposes the production dominant-result +configuration used by the Python runtime. Java additionally offers three selectable +reduction modes, more normalization controls, incremental extension, and a CLI +with explicit overwrite handling. Use [Java CLI Compilation](../cli-compilation.md) +when those controls are required. + +For normal Python use, compile once during preparation, deploy the resulting +`.rxc` file as an application-owned asset, and reuse one loaded `Stemmer` at +runtime. diff --git a/docs/python/performance.md b/docs/python/performance.md new file mode 100644 index 0000000..2e75570 --- /dev/null +++ b/docs/python/performance.md @@ -0,0 +1,160 @@ +# Performance (Python) + +This page reports **runtime stemming throughput** of the Python implementation against +common Python stemmers, and — crucially — documents exactly how the comparison +is made fair. The scripts are in the repository (`python/benchmarks/`); anyone +can reproduce the numbers. + +!!! info "Published single-machine measurement" + These results were regenerated on 2026-08-08 on the current benchmark + workstation: Fedora Linux 44 (`7.1.6-201.fc44.x86_64`), AMD Ryzen 5 5625U + (6 cores / 12 threads), CPython 3.14.6, Rust 1.97.1, and a release wheel. + All logical CPUs used the `schedutil` governor. Absolute timings remain + machine-specific; compare ratios only within this run. + +## What is measured + +- **Runtime stemming only.** Model construction / dictionary compilation happens + once in setup and is excluded from every timing. +- **Workload = the Java JMH corpus.** The *changed-token* corpus derived from + the bundled UniMorph gold-standard dictionaries: every dictionary field paired + with its line's root, normalized `trim().lower()`, keeping only tokens that + differ from their root (the forms a stemmer must actually rewrite), padded to + ≥ 5 000 tokens. This is identical to the Java `LanguageBenchmarkCorpus`. +- **Batch sizes 10/20/50/100** are swept and a line is fit to `per_call(N) = + intercept + N · slope` as a descriptive scaling summary. This is an + unconstrained OLS fit, so noise may produce a negative intercept; it is not a + physical decomposition of runtime. The *best* of many repeats is reported. + +## Fairness: making the comparison apples-to-apples + +Three asymmetries silently distort stemmer comparisons. Each is neutralized, and +where it **cannot** be neutralized the effect is described. + +1. **Result caching — neutralized.** PyStemmer caches results by default + (`maxCacheSize=10000`). Since a benchmark stems the same corpus repeatedly, + that cache would turn measured passes into dictionary lookups rather than + stemming. The harness explicitly disables **both** PyStemmer's cache + (`maxCacheSize=0`) and radixor's default cache (`cache_size=0`). The other + engines have no cache. +2. **Lowercasing — neutralized.** Snowball (PyStemmer, snowballstemmer) and + CISTEM differ in whether they case-fold. Snowball does **no** case handling; + it assumes pre-lowercased input. The corpus is pre-lowercased for every + engine, and radixor is therefore run with **`lowercase=False`** so it does + the same work. On already-lowercased input radixor returns identical results + either way. **Exception — CISTEM:** it always performs its own lowercasing + and German umlaut normalization internally and cannot be told to skip it, so + CISTEM does *slightly more* normalization work than the others. This + unavoidable extra work biases the comparison modestly **in radixor's + favour**, not CISTEM's. +3. **Hidden delegation — neutralized.** `snowballstemmer` delegates to PyStemmer + when PyStemmer is installed (they become the same C code). The harness + bypasses that and uses snowballstemmer's genuine pure-Python backend, and + records each engine's backing module + whether it is a compiled extension so + the provenance is verifiable. + +## Environment and parameters + +| Item | Published value | +|---|---| +| CPU | AMD Ryzen 5 5625U with Radeon Graphics | +| CPU topology | 6 physical cores / 12 logical CPUs | +| OS | Fedora Linux 44, kernel `7.1.6-201.fc44.x86_64` | +| CPU governor | `schedutil` on all 12 logical CPUs; boost enabled | +| Python | CPython 3.14.6 | +| Radixor | 4.1.0, release-mode ABI3 wheel, cache disabled | +| PyStemmer | 3.1.0 (`libstemmer_c` 3.1.0), cache disabled | +| snowballstemmer | 3.1.1, forced pure-Python backend | +| NLTK | 3.10.2 | +| Workload | 5,000 changed tokens per language and measurement | +| Batch sizes | 10, 20, 50, 100 | +| Timing | best of 15 measured passes after 3 warm-up passes | + +The authoritative command was: + +```bash +./gradlew pythonBenchmarkAllLanguagesBatch --rerun-tasks +``` + +It completed successfully in 3 minutes 33 seconds and emitted +the full per-size CSV and JSON reports under +`build/reports/python-benchmarks/`. + +## Results — batch size 100, cache disabled + +The table reports nanoseconds per word at `N=100` (lower is better). A dash +means that the engine has no implementation for that language. Every available +competitor was measured in the same process, with the same corpus and batch +partitioning. + +| Language | Radixor | PyStemmer (Snowball C) | CISTEM (pure Py) | snowballstemmer (pure Py) | NLTK Porter (pure Py) | +|---|---:|---:|---:|---:|---:| +| Czech (`cs`) | **224.3** | 236.6 | — | 4,835.2 | — | +| Danish (`da`) | **178.3** | 267.6 | — | 8,568.9 | — | +| German (`de`) | **230.9** | 635.5 | 3,341.9 | 33,654.1 | — | +| English (`en`) | **180.5** | 331.9 | — | 20,195.0 | 7,740.3 | +| Spanish (`es`) | **184.2** | 316.6 | — | 19,640.1 | — | +| Persian (`fa`) | **210.1** | 497.1 | — | 32,732.3 | — | +| Finnish (`fi`) | **227.8** | 258.8 | — | 12,339.5 | — | +| French (`fr`) | **234.2** | 503.7 | — | 36,161.9 | — | +| Hebrew (`he`) | **228.6** | — | — | — | — | +| Hungarian (`hu`) | **198.2** | 264.7 | — | 13,694.3 | — | +| Italian (`it`) | **170.8** | 517.0 | — | 34,504.6 | — | +| Norwegian Bokmål (`nb`) | **187.1** | 239.7 | — | 7,457.6 | — | +| Dutch (`nl`) | **187.1** | 354.8 | — | 18,148.2 | — | +| Norwegian Nynorsk (`nn`) | **168.7** | 231.2 | — | 7,489.4 | — | +| Polish (`pl`) | **194.6** | 214.5 | — | 5,282.9 | — | +| Portuguese (`pt`) | **166.9** | 293.2 | — | 21,157.2 | — | +| Russian (`ru`) | **273.4** | 414.4 | — | 15,703.8 | — | +| Swedish (`sv`) | **189.3** | 212.5 | — | 5,351.4 | — | +| Ukrainian (`uk`) | **221.5** | — | — | — | — | +| Yiddish (`yi`) | **227.5** | 624.2 | — | 33,251.6 | — | + +Radixor won all **18 / 18** direct PyStemmer comparisons. At `N=100`, its +geometric-mean speedup was **1.67×**; the largest direct advantage was **3.03×** +for Italian. Across all 20 Radixor languages, throughput ranged from **3.66 to +5.99 million words/s**. + +### CISTEM comparison for German + +The German row also provides a direct comparison with CISTEM: + +| Engine | Implementation | N=100 | vs radixor | +|---|---|---|---| +| **radixor** | Rust trie | **230.9 ns/word** | — | +| PyStemmer (de) | Snowball C | 635.5 ns/word | 2.75× slower | +| **CISTEM** | pure Python (`nltk`) | **3,341.9 ns/word** | **14.47× slower** | + +CISTEM has no batch entry point (it is a per-word Python loop), so its per-word +cost is flat across batch sizes and batching cannot amortize it. It is a compact +~40-rule German heuristic with no dictionary — a different design point that +trades coverage for simplicity. Because CISTEM's unavoidable normalization work +modestly biases the measurement in radixor's favour (point 2 above), the 14.47× +result is not a perfectly normalization-matched ratio. + +The all-language Gradle task does not measure stage-level profiling or cached +lookup performance. This page therefore does not mix such figures from an older +workstation into the published run. + +## A note on comparability of *quality* + +These are **speed** comparisons. Radixor is a **lexicon-trained transformation +stemmer**: it learns patch commands from UniMorph-grounded word–stem evidence +and can generalize those transformations beyond exact training entries. +Snowball, Porter, and CISTEM use hand-written rule systems. They produce +different stems and are not directly comparable on output; see the shared +[linguistic quality +methodology](../benchmarks/reference/linguistic-quality.md) for how stemming +quality is assessed separately from throughput. + +## Reproduce + +```bash +pip install -r python/benchmarks/requirements-bench.txt +./gradlew pythonBenchmarkAllLanguagesBatch --rerun-tasks +``` + +The run prints the machine/Python/engine versions and each engine's backing +module (provenance), and writes per-point rows (CSV) plus the full report +including environment (JSON). Methodology and fairness notes live in +`python/benchmarks/README.md`. diff --git a/docs/python/quick-start.md b/docs/python/quick-start.md new file mode 100644 index 0000000..cf51200 --- /dev/null +++ b/docs/python/quick-start.md @@ -0,0 +1,130 @@ +# Python Quick Start + +Radixor's Python implementation is a native Rust extension with a Python API. +It uses the same learned patch-command model and version 7 compiled-trie format +as the Java implementation, without requiring a JVM. + +## 1. Install the runtime and standard models + +Create an isolated environment and install Radixor: + +```bash +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +``` + +=== "PyPI" + + ```bash + python -m pip install --only-binary=:all: radixor + ``` + +=== "GitHub Releases" + + ```bash + python -m pip install --only-binary=:all: \ + --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor + ``` + +PyPI publication is pending. The GitHub option becomes live when the first +Python GitHub Releases populate the Pages-backed package index. See +[Installation and Builds](installation.md) for availability and source builds. + +The `radixor` wheel contains code. Its required +`radixor-models-standard` dependency contains 20 precompiled models. The +standard package excludes textual source dictionaries and optional PoliMorf +data, which keeps startup on the direct compiled-model path. + +## 2. Select and reuse a stemmer + +Construct a stemmer once and retain it for the lifetime of the application: + +```python +from radixor import Stemmer + +english = Stemmer("en") +polish = Stemmer("pl") + +print(english.stem("running")) # 'run' +print(polish.stem("koty")) +``` + +Short aliases such as `en`, `de`, and `pl` resolve to the documented default +model IDs. A full ID such as `us-uk-default` selects the same model explicitly. +The complete mapping is listed under [Built-in Languages](../built-in-languages.md). + +## 3. Prefer batch calls for collections + +Crossing the Python/native boundary once per collection is substantially more +efficient than a Python loop of scalar calls: + +```python +words = ["running", "studies", "better", "cars"] +stems = english.stem_batch(words) +``` + +`stem_batch()` preserves input order and returns one item per word. Entries can +be `None` when the trie has no applicable patch command. + +Repeated natural-language tokens use a bounded result cache shared by the +scalar and batch APIs. Its default capacity is 10,000 entries, matching +PyStemmer; choose another bound or pass `0` to disable it: + +```python +english = Stemmer("en", cache_size=10_000) +uncached = Stemmer("en", cache_size=0) +``` + +The cache covers `stem()`, `stemWord()`, `stem_batch()`, and `stemWords()`; +the `stem_all*()` methods are not cached. + +## 4. Migrate from PyStemmer + +Radixor exposes PyStemmer's familiar scalar and batch method names: + +```python +stemmer = Stemmer("en") + +stemmer.stemWord("running") +stemmer.stemWords(["running", "unknown_word"]) +``` + +`stemWord()` and `stemWords()` return unmatched input unchanged. This removes +the `None` fallback checks required by Radixor's original `stem()` and +`stem_batch()` methods, so most migration work is limited to the package import +and dependency change. + +## 5. Load a custom compiled model + +The standard installation covers the maintained default catalog. A custom +version 7 model can be loaded directly: + +```python +custom = Stemmer(compiled="models/domain-english.rxc") +``` + +To compile a maintained textual dictionary during a preparation step: + +```python +from radixor import compile + +compile("dictionaries/domain.tsv.gz", "models/domain-english.rxc", language="en") +``` + +Deploy the resulting `.rxc` file and load it at application startup. See +[Dictionary Compilation](model-compilation.md) for format interoperability and +the production compilation profile. + +## 6. Production checklist + +- Pin compatible `radixor` and `radixor-models-standard` releases in the + application's dependency lock. +- Construct and reuse stemmers instead of rebuilding them per request. +- Use batch calls for token collections. +- Choose `stem*` or `stemWord*` semantics deliberately for unmatched words. +- Treat custom dictionaries and compiled models as trusted application input. +- Regression-test representative vocabulary before changing model versions. + +Continue with [Installation and Builds](installation.md) for wheel/platform +details, [Usage and API](usage.md) for the complete call surface, or +[Performance](performance.md) for benchmark methodology and results. diff --git a/docs/python/usage.md b/docs/python/usage.md new file mode 100644 index 0000000..d901f56 --- /dev/null +++ b/docs/python/usage.md @@ -0,0 +1,173 @@ +# Usage and examples + +## Creating a stemmer + +```python +from radixor import Stemmer + +s = Stemmer("en") # by language code (bundled model) +s = Stemmer("us-uk-default") # by full model ID +s = Stemmer(path="my_dictionary.gz") # a custom gzipped TSV source dictionary +s = Stemmer(compiled="en.rxc") # a pre-compiled binary (instant load) +``` + +The traversal direction is derived from the language (right-to-left `fa`/`he`/`yi` +use FORWARD, all others BACKWARD); override with `backward=True|False` for a +custom `path=`. + +## Stemming a single word + +```python +s.stem("running") # 'run' +s.stem("cats") # 'cat' +s.stem("zzzzz") # None -> not reducible / unknown +``` + +`stem()` returns the single **dominant** stem, or `None`. + +!!! info "Why a known word may return itself" + A surface form that is *also* a canonical headword (e.g. an English word + that is both its own lemma and an inflection of another lemma) returns + itself, because the dominant sense is “this word is its own stem”. The + inflectional reading is still available via `stem_all()`. + +## Batch stemming — the fast path + +For anything beyond a handful of words, use the batch API. It crosses the +Python↔native boundary **once** for the whole list, which is the dominant cost +when stemming from Python. + +```python +words = ["running", "cats", "stemming", "quickly"] +s.stem_batch(words) # ['run', 'cat', 'stem', 'quick'] (None for unknowns) +``` + +```python +# Multiple candidate stems per word (ambiguity preserved): +s.stem_all("running") # e.g. ['run', 'runn'] +s.stem_all_batch(["running", "cats"]) +``` + +## PyStemmer-compatible methods + +Radixor also exposes PyStemmer's scalar and batch method names. They differ +from the native Radixor methods only when the trie cannot find a patch command: + +| Method | Recognized word | Word without a patch command | Return type | +| --- | --- | --- | --- | +| `stem(word)` | dominant stem | `None` | `str | None` | +| `stem_batch(words)` | dominant stem at the same position | `None` at the same position | `list[str | None]` | +| `stemWord(word)` | dominant stem | original input word | `str` | +| `stemWords(words)` | dominant stem at the same position | original input word at the same position | `list[str]` | + +Use `stemWord()` and `stemWords()` when migrating code that expects +PyStemmer's no-`None` contract: + +```python +import radixor as Stemmer + +# The rest of this common PyStemmer call pattern remains unchanged. +s = Stemmer.Stemmer("english") + +s.stemWord("running") # 'run' +s.stemWord("unknown_word") # 'unknown_word' +s.stemWords(["running", "unknown_word"]) +# ['run', 'unknown_word'] +``` + +`stemWords()` retains input order and makes one Python-to-Rust call for the +whole list. PyStemmer's full language names, such as `"english"` and +`"czech"`, are accepted for bundled Radixor languages alongside two-letter +codes and full model IDs. + +The compatibility contract covers these method names, full language aliases, +and unmatched-word fallback behavior. Radixor configuration keywords remain +Radixor-specific: use `cache_size`, not PyStemmer's `maxCacheSize`. Both +libraries default to a cache capacity of 10,000 entries. + +## Bounded result cache + +Real text repeats tokens. The default bounded cache returns the already-built +result object on a recognized-word hit (a reference-count bump — no +re-stemming, no new result string). Unknown words are cached as misses, so +`stemWord()` and `stemWords()` still create their required original-word +result. Its default capacity is **10,000 entries**, matching PyStemmer: + +```python +s = Stemmer("en") # cache up to 10,000 distinct input words +s = Stemmer("en", cache_size=50_000) # choose a custom capacity +s = Stemmer("en", cache_size=0) # explicitly disable caching +``` + +One cache is shared by `stem()`, `stemWord()`, `stem_batch()`, and +`stemWords()`. The `stem_all()` and `stem_all_batch()` methods are not cached. +Caching never changes results; it only avoids recomputation. Entries are +inserted until the configured capacity is reached; there is no eviction. For a +high-cardinality stream without useful token repetition, use `cache_size=0`. + +## Skipping lowercasing for pre-normalized input + +By default lookups lowercase the input (`LOWERCASE_WITH_LOCALE_ROOT`). If your +pipeline already lowercases tokens, skip the redundant work: + +```python +s = Stemmer("en", lowercase=False) # assume already-lowercased input +s.stem("running") # 'run' +s.stem("Running") # None -> not lowercased, so no match +``` + +The model's keys are always lowercase; `lowercase=False` only turns off +per-lookup normalization. On already-lowercased input the results are identical. + +## Compile once, load instantly + +Compiling a trie from text costs a few seconds for large languages. Compile it +once to Radixor's binary format and load it directly afterwards: + +```python +import radixor + +radixor.compile("stemmer.gz", "en.rxc", language="en") +s = radixor.Stemmer(compiled="en.rxc") +``` + +See [Compiling Dictionaries in Python](model-compilation.md) for the source +format, traversal and normalization options, deployment guidance, Java +interoperability, and the controls that remain Java-only. + +## Using a custom dictionary + +A source dictionary is a gzipped (or plain) TSV file, one entry per line, the +first column the canonical stem and the rest its variants; `#` and `//` start +line remarks: + +``` +run running runs ran +cat cats +``` + +```python +s = Stemmer(path="custom.gz", backward=True, store_original=True) +``` + +`store_original=True` (default) maps each stem to itself (a no-op patch) so the +stem is recognised. See [Dictionary Format](../dictionary-format.md) for the +authoritative specification shared with the Java project. + +## Thread-safety + +A `Stemmer` is safe to share across threads. The bounded cache is guarded +internally; the compiled trie is immutable after construction. + +## API summary + +| Call | Returns | Notes | +|---|---|---| +| `Stemmer(lang \| path= \| compiled=, *, backward, store_original, lowercase, cache_size=10_000)` | stemmer | auto-detects compiled vs textual for `path=`; `cache_size=0` disables caching | +| `stem(word)` | `str \| None` | dominant stem | +| `stem_batch(words)` | `list[str \| None]` | **preferred** for many words | +| `stemWord(word)` | `str` | PyStemmer-compatible; returns an unmatched word unchanged | +| `stemWords(words)` | `list[str]` | PyStemmer-compatible batch call; preserves unmatched words and input order | +| `stem_all(word)` | `list[str]` | all candidate stems, best first | +| `stem_all_batch(words)` | `list[list[str]]` | | +| `radixor.compile(source, out, *, language, backward, store_original, lowercase)` | `None` | writes a v7 binary | diff --git a/docs/quick-start.md b/docs/quick-start.md index 2d88258..71d9d74 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,6 +1,9 @@ -# Quick Start +# Java Quick Start -This guide introduces the fastest practical path to using **Radixor**. +This guide introduces the fastest practical path to using **Radixor in Java**. +Python users have an equivalent application-oriented path in the +[Python Quick Start](python/quick-start.md), followed by the +[Python Usage and API guide](python/usage.md). If you are new to Radixor and want the shortest possible path to a first working stem, start with [Fast Track](fast-track.md). This Quick Start is a broader developer walkthrough: it introduces the diff --git a/docs/technology-lineage.md b/docs/technology-lineage.md new file mode 100644 index 0000000..37d8666 --- /dev/null +++ b/docs/technology-lineage.md @@ -0,0 +1,236 @@ +# Technology and Lineage + +Radixor has an unusual position in the stemming ecosystem because its history +predates several implementations with which it is now compared. + +The useful way to describe that history is not as a list of project names, but +as a set of **algorithmic lineages and runtime models**. + +## The Egothor lineage + +The transformation-based approach behind Radixor was described by Leo Galambos +in *Lemmatizer for Document Information Retrieval Systems in JAVA* (SOFSEM +2001). The Egothor implementation developed the patch-command/trie approach used +to compile word-form transformations into a compact stemming structure. + +Radixor is a modern implementation of that lineage. It is not a binary or source +repackaging of the old project: the current implementation has a new runtime +representation, compiled patch commands, deterministic multi-result semantics, +modern reduction modes, persistence, model packaging, integrity validation, and +current Java/Python integration. + +See [Why Radixor Is Different](why-radixor-is-different.md) for the architecture +rather than the chronology. + +## Stempel is a historical Egothor branch, not an independent algorithmic lineage + +This point is easy to miss when Stempel is encountered through Lucene or +Elasticsearch. + +Lucene's own Stempel documentation states that the core stemming algorithm and +implementation were taken **verbatim / virtually unchanged from the Egothor +project**. The Stempel distribution is principally associated with its Polish +stemming tables, even though the underlying algorithm is not inherently +Polish-specific. + +That makes Stempel historically important, but it should be interpreted +correctly in Radixor comparisons: + +- it demonstrates that the older Egothor technique survived in major search + infrastructure; +- it is not evidence of a separate later algorithm that Radixor subsequently + copied; +- benchmarking Stempel against Radixor is effectively a comparison between a + preserved legacy branch of the technique and its modern re-engineering. + +Official Lucene reference: +[StempelStemmer](https://lucene.apache.org/core/9_9_1/analysis/stempel/org/apache/lucene/analysis/stempel/StempelStemmer.html). + +In the current Polish benchmark, the direct Stempel path has balanced accuracy +**0.855699** versus **0.991105** for Radixor, and its direct runtime is measured +at **4.229×** the Radixor time. The Lucene StempelFilter path is **4.803×** the +Radixor time. See the [Polish benchmark](benchmarks/languages/polish.md). + +## Morfologik is a closed-vocabulary morphological lookup + +Morfologik solves a different problem from Radixor. Its runtime is a +dictionary-driven morphological lookup backed by a finite-state automaton (FSA). +The distinction is not merely terminology: it determines what happens when +production text contains a word form that the dictionary does not know. + +`DictionaryLookup.lookup(...)` searches the compiled automaton for the supplied +surface form and returns the stored base-form analyses only when that form is +present. If the lookup fails, it returns an empty result. Lucene's +`MorfologikFilter` tries the original token and then its lowercase form; if both +lookups fail, the filter emits the original token unchanged. + +There is therefore **no rule-based or learned transformation fallback for an +out-of-vocabulary word**. + +Primary-source implementations: + +- [Morfologik `DictionaryLookup`](https://github.com/morfologik/morfologik-stemming/blob/master/morfologik-stemming/src/main/java/morfologik/stemming/DictionaryLookup.java) +- [Lucene `MorfologikFilter`](https://github.com/apache/lucene/blob/main/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java) + +This makes dictionary completeness an operational requirement, not just a +quality-tuning parameter. A new domain term, previously unseen inflection, +product name, spelling variant, or other surface form outside the compiled +dictionary receives no morphological reduction from Morfologik. In an +open-vocabulary search system, maintaining coverage therefore requires a +sufficiently comprehensive dictionary and continued dictionary updates. + +Radixor uses lexical resources differently. Its source data is **training +evidence for transformations**. Word-to-root relationships are converted into +patch commands, organized in a trie, structurally reduced, and compiled into a +runtime machine. The deployed stemmer selects transformation behaviour and +applies it to the input token; it is not restricted to retrieving a stored +analysis for an exact dictionary member. + +This architectural difference matters when interpreting quality numbers. +Morfologik can provide strong analyses for vocabulary covered by its dictionary, +but that strength does not imply generalization to unseen forms. Radixor is +designed to preserve the linguistic evidence of large lexical resources while +turning it into reusable transformation behaviour. + +The current Polish benchmark also places the two approaches at very different +points on the measured quality/performance envelope: + +- deterministic primary-output balanced accuracy is **0.991105** for Radixor + and **0.948392** for `MorfologikFilter`; +- when all emitted candidates are considered, balanced accuracy is + **1.000000** for Radixor and **0.987528** for Morfologik; +- the measured Lucene `MorfologikFilter` runtime is **15.997×** the Radixor + runtime in the same Java benchmark. + +See the [Polish benchmark](benchmarks/languages/polish.md). + +The useful conclusion is not that Morfologik is unsophisticated. It is a +morphological dictionary system with a richer analysis objective. The important +engineering distinction is sharper: **it pays the runtime and storage cost of +dictionary/FSA analysis while remaining bounded by dictionary coverage; +Radixor compiles lexical evidence into a substantially smaller hot-path +transformation problem that can also operate beyond explicitly observed word +forms.** + +## Snowball and Porter are fixed rule systems + +Porter and Snowball form another distinct lineage. Their language algorithms are +explicit rule programs, usually centered on suffix regions and ordered rewrite +rules. + +They have a genuine advantage over closed dictionary lookup: their rules +naturally apply to previously unseen words. The trade-off is that the +linguistic behaviour is encoded in the hand-designed rule program itself rather +than learned from lexical evidence. + +At `N=100`, the current Python batch benchmark shows that rule-based +generalization does not require accepting a runtime advantage over Radixor: + +- **18 / 18** direct language comparisons are won by Radixor against PyStemmer + 3.1.0; +- the geometric-mean speedup is **1.67×**; +- the largest measured direct advantage is **3.03×** (Italian); +- Radixor spans **3.66–5.99 million words/s** across all 20 measured Radixor + languages at batch size `N=100`. + +Those are performance results. The newly integrated official Snowball 3.1.0 Java +quality comparators also make the linguistic trade-off visible for the three +algorithms added in that Snowball generation: + +| Language, `ALL_WORDS` | Radixor balanced accuracy | Snowball 3.1.0 balanced accuracy | Radixor OI / UI | Snowball OI / UI | +| --- | ---: | ---: | ---: | ---: | +| Czech | **0.996617** | 0.786366 | **0% / 0.676519%** | 0.000904% / 42.725842% | +| Persian | **0.976360** | 0.535123 | **0% / 4.728041%** | 0.001278% / 92.974054% | +| Polish | **0.991105** | 0.823625 | **0% / 1.779024%** | 0.000967% / 35.273970% | + +The lowercase-only evaluation gives the same picture: + +- **Czech:** 0.997195 vs 0.784821 balanced accuracy, with UI 0.561033% vs + 43.034822%; +- **Persian:** 0.976360 vs 0.535123, with UI 4.728041% vs 92.974054%; +- **Polish:** 0.991301 vs 0.823465, with UI 1.739895% vs 35.306102%. + +The dominant difference is under-stemming rather than excessive conflation. +Snowball's over-stemming remains very low in these measurements, but it leaves +a much larger share of gold-related forms ungrouped. That distinction matters: +a conservative stemmer can look safe when judged only by false conflations +while still sacrificing substantial recall. + +Finnish remains another useful illustration. The published `ALL_WORDS` +primary-output quality result is **0.984838** balanced accuracy for Radixor +versus **0.740279** for the Snowball Finnish Lucene path, with under-stemming +**3.032474%** versus **51.944179%**. At `N=100` in the current Python batch run, +Radixor is **1.14×** faster than PyStemmer's Finnish implementation. + +See the [Finnish benchmark](benchmarks/languages/finnish.md), the +[Czech benchmark](benchmarks/languages/czech.md), the +[Persian benchmark](benchmarks/languages/persian.md), and the +[Polish benchmark](benchmarks/languages/polish.md). + +The important comparison is therefore not “dictionary versus rules”. +Radixor occupies a third position: **it learns transformations from lexical +evidence, compiles them into a reduced patch-command trie, and retains +algorithmic generalization at runtime without hardcoding a fixed suffix program.** + +## Lucene light, minimal, plural, and possessive filters + +Several Lucene language filters are intentionally narrow transformations. A +minimal or light stemmer may be extremely fast precisely because it performs +less linguistic conflation. + +That is not a defect. It is a different objective. + +The important benchmark discipline is therefore: + +> do not interpret runtime without the corresponding grouping quality. + +A stemmer that removes only a tiny set of endings and a stemmer that attempts +broad morphological conflation are not doing equivalent work merely because +both return a string called a “stem”. + +## Hunspell + +Hunspell combines dictionaries with affix rules and can produce several +candidate stems. It is another useful comparator because it occupies a middle +ground between direct dictionary analysis and pure suffix stemming. + +Its architecture is still different from Radixor's compiled patch-command trie: +Hunspell interprets lexical and affix resources, whereas Radixor has already +compiled observed transformation behaviour into a reduced runtime machine. + +## What the current benchmark results justify saying + +The project does not need to position Radixor merely as “another stemmer”. + +A more accurate statement is: + +> **Radixor is a learned transformation stemmer built around reduced +> patch-command tries. Its current public benchmarks show that this architecture +> can move the quality/performance frontier rather than merely trade one for the +> other.** + +That is an architectural and empirical claim, not a claim that every alternative +project is poorly designed. Different systems were built for different goals: + +| Family | Primary runtime idea | Typical strength | Key distinction from Radixor | +| --- | --- | --- | --- | +| Radixor | Learned patch commands in a reduced compiled trie | High-quality conflation with compact deterministic runtime | Training data compiles into reusable transformations | +| Stempel | Historical Egothor implementation + stemming tables | Proven legacy deployment, especially Polish | Same historical algorithmic lineage; older implementation branch | +| Morfologik | Dictionary/FSA morphological lookup | Rich in-vocabulary lemmatization and multiple analyses | Closed-vocabulary lookup: unknown forms have no stemming fallback; Lucene passes them through unchanged | +| Snowball / Porter | Fixed language rule programs | Portable rule-based stemming with natural OOV coverage | Rules generalize to unseen forms, but are authored rather than learned from lexical evidence | +| Lucene light/minimal | Deliberately narrow handcrafted rules | Very low runtime cost | Intentionally less linguistic work | +| Hunspell | Dictionary + affix rules | Lexical/affix analysis and candidate outputs | Runtime interprets lexicon/affix resources | + +The benchmark pages remain the authority for each language and comparator. The +purpose of this page is to make the **technology categories** explicit so readers +do not have to infer them from implementation names. + +## Historical references + +- Leo Galambos, *Lemmatizer for Document Information Retrieval Systems in JAVA* + (SOFSEM 2001) +- [Lucene StempelStemmer documentation](https://lucene.apache.org/core/9_9_1/analysis/stempel/org/apache/lucene/analysis/stempel/StempelStemmer.html) +- [Lucene Morfologik package documentation](https://lucene.apache.org/core/10_3_2/analysis/morfologik/org/apache/lucene/analysis/morfologik/package-summary.html) +- [Architecture](architecture.md) +- [Benchmark results](benchmarks/index.md) +- [Tested stemmer inventory](benchmarks/reference/tested-stemmers.md) diff --git a/docs/why-radixor-is-different.md b/docs/why-radixor-is-different.md new file mode 100644 index 0000000..c6f8b3e --- /dev/null +++ b/docs/why-radixor-is-different.md @@ -0,0 +1,161 @@ +# Why Radixor Is Different + +Radixor is **dictionary-trained, not dictionary-bound**. + +The source dictionaries are build-time evidence from which Radixor learns +word-to-stem transformations. The runtime artifact is not a flat table that can +only answer words already present in that evidence. It is a compact, +deterministic **trie of patch commands**. + +That distinction is the shortest way to understand the project. + +## A learned transformation stemmer + +A conventional dictionary lookup stores a relationship such as: + +```text +running -> run +``` + +Radixor instead derives a transformation that can be represented conceptually as: + +```text +running -> -> run +``` + +Many word forms share the same transformation behaviour. Radixor organizes those +commands in a trie, reduces structurally equivalent regions, contracts uniform +preferred-command subtrees, and freezes the result into an immutable compiled +runtime structure. + +The pipeline is therefore: + +```mermaid +flowchart TD + evidence[Lexical evidence] + transformations[Word-to-root transformations] + commands[Patch commands] + mutable[Mutable trie] + reduction[Subtree contraction and semantic reduction] + compiled[Compiled trie] + runtime[Runtime command selection and application] + + evidence --> transformations --> commands --> mutable + mutable --> reduction --> compiled --> runtime +``` + +The dictionary is important because it supplies the linguistic evidence. It does +**not** define a closed runtime vocabulary. + +## What happens to an unseen word? + +The compiled trie selects transformation behaviour rather than storing a full +lemma string for every possible input. + +Uniform subtrees can be contracted into accepting leaves. When lookup reaches an +accepting leaf whose preferred patch command is already determined, the runtime +can apply that command even with input characters remaining. This is one of the +ways the compiled model can generalize beyond explicitly observed dictionary +forms. + +Generalization is not a promise that every arbitrary unknown token has a useful +stem. No practical stemmer can make that guarantee. The important property is +that Radixor is **not limited to exact dictionary membership**. + +For the implementation details, see [Architecture](architecture.md). + +## Why patch commands matter + +Patch commands encode *how to transform* a word rather than merely *which string +to return*. + +That gives the runtime model several useful properties: + +- repeated transformation behaviour can be shared; +- trie paths share structural information between related inputs; +- equivalent subtrees can be reduced; +- the final command is applied directly to the original token; +- the runtime can expose a deterministic preferred result; +- the same compiled node may retain ranked alternative commands when ambiguity + should not be discarded. + +The result is closer to a compact learned transformation machine than to either a +flat dictionary or a handwritten suffix list. + +## Why the trie matters + +The trie is not just a storage container around a dictionary. + +It is the structure that makes the learned transformations reusable. Shared +paths represent shared input structure; reduction merges equivalent behaviour; +uniform-subtree contraction can terminate preferred-result lookup early. + +This is also why the source dictionary may be very large while the deployed +representation remains compact and fast. + +## Radixor is not three common things + +### It is not a closed dictionary lemmatizer + +A closed dictionary lemmatizer primarily asks whether the current surface form +exists in a lexicon or automaton and, if it does, returns stored analyses. + +Radixor uses lexical resources differently: it **compiles transformation +behaviour from them**. + +### It is not another fixed suffix-rule stemmer + +Porter- and Snowball-family stemmers encode explicit rules for a language. +Those systems can generalize because the rules apply to unseen text, but the +rules themselves are fixed algorithmic knowledge. + +Radixor learns its transformation behaviour from language data and then compiles +that behaviour into its runtime trie. + +### It is not a full morphological analyzer + +A full analyzer may return lemmas, parts of speech, grammatical tags, and +multiple analyses. That is valuable when applications need morphological +interpretation. + +Radixor has a narrower search-oriented objective: produce compact, high-quality +term conflation with predictable runtime cost. It can preserve multiple stemming +candidates, but it does not attempt to become a general-purpose morphological +analysis framework. + +## What is distinctive about the combination + +Any one ingredient in isolation is familiar: + +- dictionaries are familiar; +- tries are familiar; +- string edit commands are familiar; +- subtree reduction is familiar. + +The distinctive architecture is their **combination**: + +> lexical evidence → patch commands → trie organization → semantic reduction → +> compact deterministic runtime transformation + +That architecture separates expensive learning and compilation from hot-path +runtime work. + +## Modern Radixor adds more than the historical implementation + +Radixor preserves the useful Egothor idea while rebuilding the operational model +for current software: + +- immutable compiled tries; +- compiled patch-command objects rather than repeated textual interpretation; +- deterministic ranked multi-result lookup; +- configurable reduction semantics; +- uniform-subtree contraction; +- binary persistence; +- independent language-model versioning and integrity verification; +- reopening and extending compiled structures; +- Java and Python native runtimes; +- reproducible quality and performance benchmark infrastructure. + +For the historical lineage and how Stempel, Morfologik, Snowball, and other +comparators relate to this architecture, continue with +[Technology and Lineage](technology-lineage.md). diff --git a/gradle/python.gradle b/gradle/python.gradle new file mode 100644 index 0000000..9c4285c --- /dev/null +++ b/gradle/python.gradle @@ -0,0 +1,303 @@ +def pythonProjectDirectory = layout.projectDirectory.dir('python') +def pythonHostDistributionDirectory = layout.buildDirectory.dir('python/dist/host') +def pythonSdistDistributionDirectory = layout.buildDirectory.dir('python/dist/sdist') +def pythonStandardDistributionDirectory = layout.buildDirectory.dir('python/dist/standard') +def pythonGeneratedStandardProjectDirectory = layout.buildDirectory.dir('python/generated/models-standard') +def pythonBenchmarkRuntimeDirectory = layout.buildDirectory.dir('python/runtime/benchmark') +def pythonModelCompilerRuntimeDirectory = layout.buildDirectory.dir('python/runtime/model-compiler') +def pythonBenchmarkReportDirectory = layout.buildDirectory.dir('reports/python-benchmarks') +def pythonTemporaryDirectory = layout.buildDirectory.dir('python/tmp') + +def hostOsName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT) +def hostPlatform = hostOsName.contains('win') ? 'windows' + : hostOsName.contains('mac') || hostOsName.contains('darwin') ? 'macos' + : 'linux' +def hostArchitectureName = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT) +def hostArchitecture = hostArchitectureName in ['aarch64', 'arm64'] ? 'aarch64' : 'x86_64' + +def pythonExecutable = providers.gradleProperty('pythonExecutable') + .orElse(hostPlatform == 'windows' ? 'python' : 'python3') +def maturinExecutable = providers.gradleProperty('maturinExecutable').orElse('maturin') +def pythonToolIdentity = providers.exec { + commandLine(pythonExecutable.get(), '--version') +}.standardOutput.asText.map { String value -> value.strip() } +def maturinToolIdentity = providers.exec { + commandLine(maturinExecutable.get(), '--version') +}.standardOutput.asText.map { String value -> value.strip() } +def rustToolIdentity = providers.exec { + commandLine('rustc', '--version') +}.standardOutput.asText.map { String value -> value.strip() } +def pythonBenchmarkWords = providers.gradleProperty('pythonBenchmarkWords').orElse('5000') +def pythonBenchmarkRepeats = providers.gradleProperty('pythonBenchmarkRepeats').orElse('15') +def pythonBenchmarkWarmup = providers.gradleProperty('pythonBenchmarkWarmup').orElse('3') + +def rustTargets = [ + linux : [x86_64: 'x86_64-unknown-linux-gnu', aarch64: 'aarch64-unknown-linux-gnu'], + windows: [x86_64: 'x86_64-pc-windows-msvc', aarch64: 'aarch64-pc-windows-msvc'], + macos : [x86_64: 'x86_64-apple-darwin', aarch64: 'aarch64-apple-darwin'] +] + +def pythonBuildStandardModels = tasks.register('pythonBuildStandardModels', Exec) { + group = 'python' + description = 'Builds the pure py3-none-any standard-model wheel and offline-ready sdist.' + + dependsOn('regeneratePythonStandardModels') + inputs.dir(pythonGeneratedStandardProjectDirectory) + inputs.file(pythonProjectDirectory.file('scripts/build_standard_distribution.py')) + inputs.property('pythonExecutable', pythonExecutable) + inputs.property('pythonToolIdentity', pythonToolIdentity) + outputs.dir(pythonStandardDistributionDirectory) + + workingDir(layout.projectDirectory) + doFirst { + final File output = pythonStandardDistributionDirectory.get().asFile + if (output.exists() && !output.deleteDir()) { + throw new GradleException("Cannot clean standard-model distribution directory: ${output}") + } + output.mkdirs() + commandLine(pythonExecutable.get(), 'python/scripts/build_standard_distribution.py', + '--project', pythonGeneratedStandardProjectDirectory.get().asFile.absolutePath, + '--outdir', output.absolutePath) + } +} + +def pythonBuildSdist = tasks.register('pythonBuildSdist', Exec) { + group = 'python' + description = 'Builds the Radixor native source distribution without runtime model data.' + + inputs.files(pythonProjectDirectory.file('Cargo.toml'), pythonProjectDirectory.file('Cargo.lock'), + pythonProjectDirectory.file('pyproject.toml'), layout.projectDirectory.file('gradle/python.gradle')) + inputs.dir(pythonProjectDirectory.dir('src')) + inputs.files(fileTree(pythonProjectDirectory.dir('radixor')) { + exclude 'models/**' + }) + inputs.property('maturinExecutable', maturinExecutable) + inputs.property('maturinToolIdentity', maturinToolIdentity) + outputs.dir(pythonSdistDistributionDirectory) + + workingDir(pythonProjectDirectory) + doFirst { + final File output = pythonSdistDistributionDirectory.get().asFile + if (output.exists() && !output.deleteDir()) { + throw new GradleException("Cannot clean Python sdist output directory: ${output}") + } + output.mkdirs() + commandLine(maturinExecutable.get(), 'sdist', '--out', output.absolutePath) + } +} + +def registerPythonWheelBuild = { String taskName, String taskDescription, Provider outputDirectory, + String target -> + tasks.register(taskName, Exec) { + group = 'python' + description = taskDescription + + inputs.files(pythonProjectDirectory.file('Cargo.toml'), pythonProjectDirectory.file('Cargo.lock'), + pythonProjectDirectory.file('pyproject.toml'), layout.projectDirectory.file('gradle/python.gradle')) + inputs.dir(pythonProjectDirectory.dir('src')) + inputs.dir(pythonProjectDirectory.dir('radixor')) + inputs.property('maturinExecutable', maturinExecutable) + inputs.property('maturinToolIdentity', maturinToolIdentity) + inputs.property('rustToolIdentity', rustToolIdentity) + inputs.property('pythonBuildTarget', target == null ? 'host' : target) + if (target == null) { + inputs.property('pythonExecutable', pythonExecutable) + inputs.property('pythonToolIdentity', pythonToolIdentity) + } + outputs.dir(outputDirectory) + + workingDir(pythonProjectDirectory) + doFirst { + final File output = outputDirectory.get().asFile + if (output.exists() && !output.deleteDir()) { + throw new GradleException("Cannot clean Python wheel output directory: ${output}") + } + output.mkdirs() + final List arguments = [ + 'build', '--release', '--locked', '--out', output.absolutePath + ] + if (target != null) { + arguments.addAll(['--target', target]) + } else { + arguments.addAll(['--interpreter', pythonExecutable.get()]) + } + commandLine([maturinExecutable.get()] + arguments) + } + } +} + +def pythonBuildNativeWheel = registerPythonWheelBuild( + 'pythonBuildNativeWheel', + 'Builds the Radixor native Python wheel for the current host platform.', + pythonHostDistributionDirectory, + null +) +def pythonBuild = tasks.register('pythonBuild') { + group = 'python' + description = 'Builds the host native wheel/sdist and generated standard-model wheel/sdist.' + dependsOn(pythonBuildNativeWheel, pythonBuildStandardModels, pythonBuildSdist) +} + +rustTargets.each { String platform, Map architectureTargets -> + final String taskName = 'pythonBuild' + platform.capitalize() + if (platform == hostPlatform) { + tasks.register(taskName) { + group = 'python' + description = "Builds the Radixor Python release wheel for ${platform} on the current host." + dependsOn(pythonBuild) + } + } else { + final String propertyName = 'python' + platform.capitalize() + 'Target' + final String target = providers.gradleProperty(propertyName) + .getOrElse(architectureTargets[hostArchitecture]) + registerPythonWheelBuild( + taskName, + "Cross-builds the Radixor Python release wheel for ${platform}; requires the target toolchain.", + layout.buildDirectory.dir("python/dist/${platform}"), + target + ) + } +} + +tasks.register('preparePythonBenchmarkRuntime', Sync) { + group = 'python' + description = 'Extracts the host Python wheel into an isolated benchmark runtime.' + + dependsOn(pythonBuild) + + from { + final Set wheels = fileTree(pythonHostDistributionDirectory).matching { + include '*.whl' + }.files + if (wheels.size() != 1) { + throw new GradleException("Expected exactly one host Python wheel, found ${wheels.size()} in " + + pythonHostDistributionDirectory.get().asFile) + } + zipTree(wheels.first()) + } + from { + final Set wheels = fileTree(pythonStandardDistributionDirectory).matching { + include '*.whl' + }.files + if (wheels.size() != 1) { + throw new GradleException("Expected exactly one standard-model Python wheel, found ${wheels.size()} in " + + pythonStandardDistributionDirectory.get().asFile) + } + zipTree(wheels.first()) + } + into(pythonBenchmarkRuntimeDirectory) +} + +tasks.register('preparePythonModelCompilerRuntime', Sync) { + group = 'python' + description = 'Extracts the host native wheel for deterministic standard-model regeneration.' + dependsOn(pythonBuildNativeWheel) + from { + final Set wheels = fileTree(pythonHostDistributionDirectory).matching { + include '*.whl' + }.files + if (wheels.size() != 1) { + throw new GradleException("Expected exactly one host Python wheel, found ${wheels.size()} in " + + pythonHostDistributionDirectory.get().asFile) + } + zipTree(wheels.first()) + } + into(pythonModelCompilerRuntimeDirectory) +} + +tasks.register('regeneratePythonStandardModels', Exec) { + group = 'python' + description = 'Generates standard .rxc artifacts and metadata below build/ from canonical model sources.' + dependsOn(tasks.named('preparePythonModelCompilerRuntime')) + inputs.file(layout.projectDirectory.file('models/model-projects.properties')) + inputs.file(layout.projectDirectory.file('models/catalog-version.txt')) + inputs.files(fileTree(layout.projectDirectory.dir('models')) { + include '*/build.gradle', '*/model-version.txt', '*/src/modelInput/stemmer.gz', + '*/src/modelInput/NOTICE-model-data.txt' + }) + inputs.files(fileTree(pythonProjectDirectory.dir('models-standard')) { + exclude 'build/**', 'dist/**', '*.egg-info/**', '**/__pycache__/**', + 'radixor_models_standard/manifest.json', 'radixor_models_standard/models/*.rxc', + 'radixor_models_standard/notices/*/NOTICE-model-data.txt' + }) + inputs.file(pythonProjectDirectory.file('scripts/build_standard_models.py')) + outputs.dir(pythonGeneratedStandardProjectDirectory) + workingDir(layout.projectDirectory) + doFirst { + environment('PYTHONPATH', pythonModelCompilerRuntimeDirectory.get().asFile.absolutePath) + commandLine( + pythonExecutable.get(), 'python/scripts/build_standard_models.py', + '--project', pythonGeneratedStandardProjectDirectory.get().asFile.absolutePath, + '--distribution-version', '0.0.0' + ) + } +} + +tasks.register('pythonVerifyDistributions', Exec) { + group = 'verification' + description = 'Verifies both Python archives and a fresh offline wheel-only installation.' + dependsOn(pythonBuild) + inputs.dir(pythonHostDistributionDirectory) + inputs.dir(pythonSdistDistributionDirectory) + inputs.dir(pythonStandardDistributionDirectory) + inputs.file(pythonProjectDirectory.file('scripts/verify_distributions.py')) + outputs.upToDateWhen { false } + workingDir(layout.projectDirectory) + commandLine( + pythonExecutable.get(), 'python/scripts/verify_distributions.py', + '--main-wheel-dir', pythonHostDistributionDirectory.get().asFile.absolutePath, + '--main-sdist-dir', pythonSdistDistributionDirectory.get().asFile.absolutePath, + '--standard-dir', pythonStandardDistributionDirectory.get().asFile.absolutePath + ) +} + +tasks.register('pythonBenchmarkAllLanguagesBatch', Exec) { + group = 'verification' + description = 'Benchmarks the host Python wheel and available competitors for every supported language.' + + dependsOn(tasks.named('preparePythonBenchmarkRuntime')) + + inputs.files(pythonProjectDirectory.file('benchmarks/run_benchmark.py'), + pythonProjectDirectory.file('benchmarks/corpus.py'), + pythonProjectDirectory.file('benchmarks/engines.py'), + layout.projectDirectory.file('gradle/python.gradle')) + inputs.dir(pythonBenchmarkRuntimeDirectory) + inputs.property('pythonExecutable', pythonExecutable) + inputs.property('pythonToolIdentity', pythonToolIdentity) + inputs.property('pythonBenchmarkWords', pythonBenchmarkWords) + inputs.property('pythonBenchmarkRepeats', pythonBenchmarkRepeats) + inputs.property('pythonBenchmarkWarmup', pythonBenchmarkWarmup) + outputs.file(pythonBenchmarkReportDirectory.map { it.file('all-languages-batch.csv') }) + outputs.file(pythonBenchmarkReportDirectory.map { it.file('all-languages-batch.json') }) + outputs.upToDateWhen { false } + + workingDir(pythonProjectDirectory) + doFirst { + final File reportDirectory = pythonBenchmarkReportDirectory.get().asFile + reportDirectory.mkdirs() + environment('PYTHONPATH', pythonBenchmarkRuntimeDirectory.get().asFile.absolutePath) + commandLine( + pythonExecutable.get(), + 'benchmarks/run_benchmark.py', + '--all-languages', + '--sizes', '10', '20', '50', '100', + '--words', pythonBenchmarkWords.get(), + '--repeats', pythonBenchmarkRepeats.get(), + '--warmup', pythonBenchmarkWarmup.get(), + '--csv', new File(reportDirectory, 'all-languages-batch.csv').absolutePath, + '--json', new File(reportDirectory, 'all-languages-batch.json').absolutePath + ) + } +} + +tasks.withType(Exec).configureEach { Exec task -> + if (task.name.startsWith('python') || task.name == 'regeneratePythonStandardModels') { + task.doFirst { + final File temporaryDirectory = pythonTemporaryDirectory.get().asFile + temporaryDirectory.mkdirs() + environment('TMPDIR', temporaryDirectory.absolutePath) + environment('TEMP', temporaryDirectory.absolutePath) + environment('TMP', temporaryDirectory.absolutePath) + } + } +} diff --git a/gradle/snowball-benchmarks.gradle b/gradle/snowball-benchmarks.gradle index c99b510..b6f5133 100644 --- a/gradle/snowball-benchmarks.gradle +++ b/gradle/snowball-benchmarks.gradle @@ -1,7 +1,10 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder +import java.security.MessageDigest -def snowballVersion = '3.0.1' + +def snowballVersion = '3.1.0' +def snowballArchiveSha256 = '5dab34d491f55f47b6e971569ffe6aadf5991512c648ddfe5d331b494cf6d655' def snowballArchiveName = "libstemmer_java-${snowballVersion}.tar.gz" def snowballDistributionDirectoryName = "libstemmer_java-${snowballVersion}" def snowballRootRelativePath = 'third-party/snowball' @@ -73,11 +76,32 @@ tasks.register('downloadSnowballJava') { } } +tasks.register('verifySnowballJava') { + group = 'verification' + description = 'Verifies the official Snowball Java source distribution checksum.' + + dependsOn(tasks.named('downloadSnowballJava')) + inputs.file(snowballDownloadFile) + inputs.property('expectedSha256', snowballArchiveSha256) + + doLast { + final File archive = snowballDownloadFile.get().asFile + final String actualSha256 = MessageDigest.getInstance('SHA-256') + .digest(archive.bytes) + .encodeHex() + .toString() + if (actualSha256 != snowballArchiveSha256) { + throw new GradleException("Snowball ${snowballVersion} archive SHA-256 mismatch: " + + "expected ${snowballArchiveSha256}, found ${actualSha256}.") + } + } +} + tasks.register('extractSnowballJava', Copy) { group = 'build setup' description = 'Extracts the official Snowball Java source distribution.' - dependsOn(tasks.named('downloadSnowballJava')) + dependsOn(tasks.named('verifySnowballJava')) from(tarTree(resources.gzip(snowballDownloadFile))) into(snowballExtractDirectory) diff --git a/mkdocs.yml b/mkdocs.yml index 5013e9b..6fd49ae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,15 +1,17 @@ site_name: Radixor -site_description: High-performance multi-language stemming toolkit for Java +site_description: Learned transformation stemming with compiled patch-command tries for Java and Python site_url: https://leogalambos.github.io/Radixor/ repo_url: https://github.com/leogalambos/Radixor repo_name: leogalambos/Radixor -copyright: "© 2026 Egothor. Licensed under BSD-3-Clause." +copyright: "© 2026 Egothor · Radixor software is BSD-3-Clause licensed." theme: name: material language: en + custom_dir: docs/overrides + logo: assets/images/radixor-logo.png + favicon: assets/images/radixor-logo.png features: - - navigation.instant - navigation.sections - navigation.top - search.suggest @@ -17,14 +19,20 @@ theme: - content.code.copy palette: - scheme: default - primary: indigo - accent: indigo + primary: white + accent: blue -extra: +extra: generator: false extra_css: - assets/stylesheets/extra.css + - assets/stylesheets/landing-v2.css + - assets/stylesheets/radixor-docs-safety.css + +extra_javascript: + - https://unpkg.com/mermaid@11/dist/mermaid.min.js + - assets/javascripts/mermaid.js markdown_extensions: - admonition @@ -32,18 +40,28 @@ markdown_extensions: - md_in_html - pymdownx.details - pymdownx.highlight - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true - tables nav: - Home: index.md - Start: - - Fast Track: fast-track.md - - Quick Start: quick-start.md - - Integration Deep Dive: integration-deep-dive.md + - Choose a Runtime: getting-started.md + - Why Radixor Is Different: why-radixor-is-different.md + - Python Fast Track: python/fast-track.md + - Java Fast Track: fast-track.md + - Python Quick Start: python/quick-start.md + - Java Quick Start: quick-start.md - - Integration: + - Java: + - Integration Deep Dive: integration-deep-dive.md - Overview: programmatic-usage.md - Model Selection and Loading: model-selection-and-loading.md - Loading and Building Stemmers: programmatic-loading-and-building.md @@ -52,6 +70,15 @@ nav: - Migration and Backward Compatibility: migration-and-backward-compatibility.md - CLI Compilation: cli-compilation.md + - Python: + - Fast Track: python/fast-track.md + - Quick Start: python/quick-start.md + - Overview: python/index.md + - Installation and Builds: python/installation.md + - Usage and API: python/usage.md + - Dictionary Compilation: python/model-compilation.md + - Benchmarks: python/performance.md + - Dictionaries and Languages: - Stemmer Models: stemmer-models.md - Published Model Catalog: stemmer-model-catalog.md @@ -60,6 +87,7 @@ nav: - Contributing Dictionaries: contributing-dictionaries.md - Architecture and Semantics: + - Technology and Lineage: technology-lineage.md - Overview: architecture-and-reduction.md - Architecture: architecture.md - Reduction Semantics: reduction-semantics.md diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..16cf713 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,23 @@ +# Rust and maturin build products +/target/ +**/dist/ +/models-standard/build/ +/radixor/models/ +*.so +*.egg-info/ + +# Standard-model payload is generated deterministically below the repository +# build directory and belongs only in the published wheel/sdist. +/models-standard/radixor_models_standard/manifest.json +/models-standard/radixor_models_standard/models/*.rxc +/models-standard/radixor_models_standard/notices/*/NOTICE-model-data.txt + +# Local Python environments and caches +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ + +# Reports produced by direct benchmark-script runs +/benchmarks/results*.csv +/benchmarks/results*.json diff --git a/python/Cargo.lock b/python/Cargo.lock new file mode 100644 index 0000000..86a59df --- /dev/null +++ b/python/Cargo.lock @@ -0,0 +1,254 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radixor" +version = "0.0.0" +dependencies = [ + "flate2", + "pyo3", + "unicode-general-category", + "unicode-normalization", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-general-category" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/python/Cargo.toml b/python/Cargo.toml new file mode 100644 index 0000000..024b319 --- /dev/null +++ b/python/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "radixor" +version = "0.0.0" +edition = "2021" + +[lib] +name = "_radixor" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.22", features = ["extension-module", "abi3-py39"] } +flate2 = "1.0" +unicode-normalization = "0.1" +unicode-general-category = "0.6" + +[profile.release] +lto = true +codegen-units = 1 +opt-level = 3 +strip = true diff --git a/python/LICENSE b/python/LICENSE new file mode 100644 index 0000000..b752302 --- /dev/null +++ b/python/LICENSE @@ -0,0 +1,28 @@ +Copyright (C) 2026, Leo Galambos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..0ee3a7c --- /dev/null +++ b/python/README.md @@ -0,0 +1,253 @@ +# radixor — Fastest Stemming for Python + +**radixor** is a Python extension for the [Radixor](https://github.com/leogalambos/Radixor) stemmer library, built on a Rust core via [PyO3](https://pyo3.rs/). It provides sub-microsecond per-word stemming with a batch API that amortises the Python↔Rust bridge overhead across thousands of words at once. + +## Why radixor? + +| Library | Approach | Batch API | +|---|---|---| +| **radixor** | Compiled patch-command trie in Rust | ✅ `stem_batch()` | +| PyStemmer (Snowball) | C extension (`libstemmer`) | ✅ `stemWords()` | +| snowballstemmer | Pure-Python Snowball | ✅ (Python loop) | +| NLTK Porter / CISTEM | Pure Python | ❌ | + +**Performance.** On the shared UniMorph gold-standard corpus, measuring runtime +stemming only (construction excluded) and with a fair, cache-disabled, +same-input methodology, radixor won all **18 / 18** direct comparisons with +PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run. +At batch size 100, the geometric-mean speedup was **1.67×**. The complete +machine metadata and current results are in the [Python performance +documentation](../docs/python/performance.md); benchmark implementation and +fairness notes are in [`benchmarks/`](benchmarks/README.md). + +## Installation + +From PyPI, once publication is enabled: + +```bash +python -m pip install --only-binary=:all: radixor +``` + +The GitHub Releases-backed index is the independent alternative: + +```bash +python -m pip install --only-binary=:all: \ + --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor +``` + +The GitHub command becomes usable after the first model and native releases +populate that index. See the [installation guide](../docs/python/installation.md) +for current availability and source-checkout builds. + +Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install +also resolves the mandatory pure `radixor-models-standard` dependency +with 20 precompiled standard models. Building the native source distribution +requires Rust ≥ 1.75 and [maturin](https://www.maturin.rs/). + +## Quick start + +```python +from radixor import Stemmer + +s = Stemmer("en") # English (us-uk-default model) +s.stem("running") # → "run" +s.stem("cats") # → "cat" +s.stem("unknown_word") # → None +``` + +## Batch API — the fast path + +```python +words = ["running", "cats", "stemming", "quickly"] + +# Amortises the Python→Rust bridge cost across all words at once +stems = s.stem_batch(words) +# → ["run", "cat", "stem", "quick"] +``` + +For large corpora (tens of thousands of words) the batch call is the recommended interface. It avoids per-call Python frame overhead and keeps the hot loop entirely inside Rust. + +## Migrating from PyStemmer + +Radixor provides PyStemmer's `stemWord` and `stemWords` method names. These +compatibility methods also follow PyStemmer's fallback behavior: when the trie +has no patch command, they return the original word instead of `None`. + +```python +# PyStemmer: import Stemmer +import radixor as Stemmer + +stemmer = Stemmer.Stemmer("english") +stemmer.stemWord("running") # → "run" +stemmer.stemWord("unknown_word") # → "unknown_word" +stemmer.stemWords(["running", "unknown"]) # → ["run", "unknown"] +``` + +The original Radixor methods remain unchanged: `stem` and `stem_batch` return +`None` for words without a matching patch command. Radixor accepts PyStemmer's +full language names for the languages represented by its bundled models, as +well as its existing two-letter codes and model IDs. + +## Supported languages + +| Code | Language | Model ID | +|---|---|---| +| `cs` | Czech | `cs-cz-default` | +| `da` | Danish | `da-dk-default` | +| `de` | German | `de-de-default` | +| `en` | English | `us-uk-default` | +| `es` | Spanish | `es-es-default` | +| `fa` | Persian | `fa-ir-default` | +| `fi` | Finnish | `fi-fi-default` | +| `fr` | French | `fr-fr-default` | +| `he` | Hebrew | `he-il-default` | +| `hu` | Hungarian | `hu-hu-default` | +| `it` | Italian | `it-it-default` | +| `nb` | Norwegian Bokmål | `nb-no-default` | +| `nl` | Dutch | `nl-nl-default` | +| `nn` | Norwegian Nynorsk | `nn-no-default` | +| `pl` | Polish | `pl-pl-unimorph` | +| `pt` | Portuguese | `pt-pt-default` | +| `ru` | Russian | `ru-ru-default` | +| `sv` | Swedish | `sv-se-default` | +| `uk` | Ukrainian | `uk-ua-default` | +| `yi` | Yiddish | `yi-default` | + +## API reference + +### `Stemmer(language=None, *, path=None, compiled=None, backward=None, store_original=True, lowercase=True, cache_size=10_000)` + +Create a stemmer for the given language code, model ID, custom textual +dictionary, or previously compiled version 7 trie. Textual dictionaries are +compiled in Rust; `compiled=` loads a prepared binary directly. + +```python +s = Stemmer("de") # by language code +s = Stemmer("de-de-default") # by model ID +s = Stemmer(path="/data/custom.gz") # custom gzipped dictionary +s = Stemmer(compiled="/data/custom.rxc") # prepared v7 binary +``` + +`backward` selects the traversal direction; when left as `None` it is derived +from the language (BACKWARD, except right-to-left `fa`/`he`/`yi` which use +FORWARD). `store_original` (default `True`) maps each canonical stem to a no-op +patch so the stem itself is recognised. `lowercase=False` skips runtime +lowercasing for already-normalized input, and `cache_size` enables the bounded +result cache. The default holds up to 10,000 entries, matching PyStemmer; +`cache_size=0` disables it. One cache is shared by `stem()`, `stemWord()`, +`stem_batch()`, and `stemWords()`; the `stem_all*()` methods are not cached. + +### `stem(word: str) → str | None` + +Return the stem, or `None` when the compiled trie finds no applicable patch +command. This does not mean that lookup is restricted to exact training words. + +### `stem_batch(words: list[str]) → list[str | None]` + +Stem an entire list. Preferred for large inputs. + +### `stemWord(word: str) → str` + +PyStemmer-compatible scalar method. Return the original word if it cannot be +stemmed. + +### `stemWords(words: list[str]) → list[str]` + +PyStemmer-compatible batch method. Return each unrecognized word unchanged. + +### `stem_all(word: str) → list[str]` + +Return all stems ordered by descending corpus frequency. Useful when multiple valid stems exist. + +### `stem_all_batch(words: list[str]) → list[list[str]]` + +Return all stems for each word in a batch. + +## Compiling a model (compile once, load instantly) + +Compiling the trie from a textual dictionary takes time for large languages +(seconds). You can compile it **once** to Radixor's binary format and then load +it near-instantly — the same workflow Java users have: + +```python +import radixor +radixor.compile("stemmer.gz", "en.rxc", language="en") # or backward=True/False +s = radixor.Stemmer(compiled="en.rxc") # instant load, no re-compile +``` + +The compiled file uses Radixor's **v7 trie format and is byte-compatible with +the Java `StemmerPatchTrieBinaryIO`** (the inner stream is identical), so a file +compiled by Java can be loaded by Python and vice versa. `Stemmer(path=...)` +auto-detects whether it was given a compiled trie or a textual dictionary. + +## Using a custom model + +Provide your own gzipped source dictionary (tab-separated +`stemvariant1variant2…` per line, `#` / `//` line remarks allowed) and +load it directly: + +```python +s = Stemmer(path="my_dictionary.gz") # BACKWARD by default +s = Stemmer(path="my_rtl_dictionary.gz", backward=False) # right-to-left +``` + +The dictionary is compiled to a patch-command trie in Rust at construction time. + +## Building from source + +```bash +cd Radixor/ +pip install maturin build setuptools wheel pytest +./gradlew pythonBuildStandardModels +pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl +cd python/ +maturin develop --release # editable install with release optimisations +``` + +From the repository root, Gradle builds the native wheel/sdist and pure +standard-model wheel/sdist without installing them globally: + +```bash +./gradlew pythonBuild +``` + +The convenience tasks `pythonBuildLinux`, `pythonBuildWindows`, and +`pythonBuildMacos` use the host build when the requested platform matches the +current system. Other platforms are cross-compiled with the corresponding Rust +target and therefore require that target and its linker/SDK to be installed. +Override a default target with, for example, +`-PpythonWindowsTarget=x86_64-pc-windows-gnu`. Build artifacts are written below +`build/python/dist/`. + +The complete batch benchmark runs Radixor for all bundled languages and every +available comparison engine for the languages it supports, using batch sizes +10, 20, 50, and 100: + +Comparison engines are auto-detected in the environment of `pythonExecutable`. +Install `python/benchmarks/requirements-bench.txt` there to enable the complete +comparison set. + +```bash +./gradlew pythonBenchmarkAllLanguagesBatch +``` + +Use `pythonBenchmarkWords`, `pythonBenchmarkRepeats`, and +`pythonBenchmarkWarmup` Gradle properties to tune the run. CSV and JSON reports +are written below `build/reports/python-benchmarks/`. + +Neither runtime distribution contains textual dictionaries. The standard data +sdist contains build-ready gzip v7 `.rxc` files, a checksummed provenance +manifest, and per-model CC BY-SA 3.0 notices. They are generated below `build/` +from canonical `models/*/src/modelInput/stemmer.gz` inputs and are never stored +in Git. `./gradlew regeneratePythonStandardModels` performs this deterministic +generation; repository topology selects the 20 defaults and excludes optional +`pl-pl-polimorf`. + +`radixor` requires `radixor-models-standard>=1.0,<2.0`. The Python distribution +version is independent of its `2026.1` Java model-catalog identity and of the +individual model versions recorded in the manifest. + +## License + +The native/API package is BSD-3-Clause — see [LICENSE](LICENSE). Model data +is separately licensed under CC BY-SA 3.0 in its packaged notices. diff --git a/python/benchmarks/README.md b/python/benchmarks/README.md new file mode 100644 index 0000000..5017ecd --- /dev/null +++ b/python/benchmarks/README.md @@ -0,0 +1,111 @@ +# Runtime stemming benchmarks + +These scripts measure **runtime stemming throughput only** — model construction +and dictionary compilation happen once during setup and are **excluded** from +every timing. Anyone can reproduce the numbers on their own machine. + +## What is measured, and why batch sizes + +Each engine is driven through its batch entry point over a fixed word budget +(default 5 000 tokens), split into batches of **10, 20, 50, 100** words. The +harness fits the descriptive line `per_call(N) ≈ intercept + N · slope` across +those sizes. The fit is unconstrained and timing noise can make its intercept +negative, so it describes observed scaling rather than physically separating +overhead from word work. We report the *best* (minimum) of many repeats — the +microbenchmark convention that suppresses OS/GC noise. + +## Data — identical to the Java JMH benchmarks + +The workload is the **changed-token corpus** built from the repository's +canonical dictionaries (`models//src/modelInput/stemmer.gz`), +mirroring `LanguageBenchmarkCorpus` in the Java project: each dictionary field +is paired with its line's root, normalized `trim().lower()`, and only tokens +that **differ** from their root are kept, in dictionary order, padded to ≥ 5 000 +tokens. See `corpus.py`. + +## Fairness — three things that quietly break stemmer comparisons + +Getting a *fair* comparison turned out to matter more than any micro-optimization. +Three asymmetries, if left in, make the numbers meaningless: + +1. **Result caching.** PyStemmer caches results by default (`maxCacheSize=10000`). + Because a benchmark stems the same corpus every repeat, that cache turns + measured passes into dict lookups rather than stemming. **The harness + explicitly disables both caches**: PyStemmer uses `maxCacheSize=0` and + radixor uses `cache_size=0`, so both engines do real stemming. + snowballstemmer-pure, nltk-porter, and cistem have no cache. + +2. **Lowercasing.** Snowball/PyStemmer do no case handling — they assume the + caller pre-lowercased the input (our corpus is pre-lowercased for everyone). + radixor normally lowercases internally; for a same-work comparison the + harness runs radixor with **`lowercase=False`** (assume-already-lowercased), + so Snowball and radixor do identical normalization work on identical input. + CISTEM always performs its own lowercasing and German umlaut normalization; + that unavoidable extra work modestly biases its comparison in radixor's + favour. + +3. **Delegation.** `snowballstemmer` delegates to PyStemmer when it is installed + (they become the same C code). The harness bypasses that and uses + snowballstemmer's genuine pure-Python backend, and records each engine's + backing module + whether it is a compiled extension (`--json`) as proof. + +## Engines compared + +| Engine | Implementation | Batch API | +|---|---|---| +| `radixor` | Rust patch-command trie (cache disabled) | `stem_batch` — one FFI call per batch | +| `PyStemmer` | Snowball C `libstemmer` (cache disabled) | `stemWords(list)` — one C call per batch | +| `snowballstemmer-pure` | Official **pure-Python** Snowball | `stemWords(list)` — Python loop | +| `nltk-porter` | Porter (English), pure Python | scalar loop | +| `cistem` | CISTEM (German), pure Python (`nltk`) | scalar loop | + +## Reproduce + +```bash +cd python/ +maturin develop --release # build the radixor extension into your env +cd .. +./gradlew pythonBuildStandardModels # generate and package standard models +pip install --no-deps build/python/dist/standard/radixor_models_standard-0.0.0-py3-none-any.whl +cd python/ +pip install -r benchmarks/requirements-bench.txt +python benchmarks/run_benchmark.py --language en de fr ru fi \ + --sizes 10 20 50 100 --repeats 21 \ + --json benchmarks/results.json --csv benchmarks/results.csv +``` + +From the repository root, the Gradle integration builds an isolated host wheel +and benchmarks Radixor plus every available comparison engine over all +supported languages with the fixed 10/20/50/100 size sweep: + +```bash +./gradlew pythonBenchmarkAllLanguagesBatch +``` + +The generated CSV and JSON reports are placed in +`build/reports/python-benchmarks/`. + +Comparison engines are auto-detected in the environment selected by the Gradle +`pythonExecutable` property. Install `requirements-bench.txt` in that environment +to enable the complete comparison set. + +The run prints machine/Python/engine versions and each engine's backing module, +and writes per-point rows (CSV) plus the full report incl. environment and +provenance (JSON), so results are self-describing and verifiable. + +## Published results + +The canonical, current single-machine results and complete environment metadata +are published on the documentation site's [Python performance +page](../../docs/python/performance.md). Keeping the measured table in one place +prevents results from different CPUs or benchmark runs from being mixed. + +## Interpretation + +- In the published 2026-08-08 run, **radixor is the fastest stemmer measured in + Python** in all 18 languages directly shared with PyStemmer. +- The benchmark intentionally disables caches. Cached-operation performance is + outside this suite and must not be inferred from its results. +- radixor and Snowball remain different *classes* of stemmer: radixor is + dictionary-based (UniMorph gold coverage), Snowball is rule-based. radixor + gives dictionary-quality stems *and* the best measured throughput. diff --git a/python/benchmarks/corpus.py b/python/benchmarks/corpus.py new file mode 100644 index 0000000..9758fc6 --- /dev/null +++ b/python/benchmarks/corpus.py @@ -0,0 +1,103 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Deterministic benchmark corpus, mirroring the Java JMH LanguageBenchmarkCorpus. + +The timing workload contains dictionary forms that differ from their canonical +root. Each field is normalized with ``strip().lower()``, retained when it needs +stemming, and repeated in stable dictionary order until the corpus reaches +``MINIMUM_TIMING_TOKEN_COUNT``. This mirrors +``LanguageBenchmarkCorpus.buildChangedTimingCorpus`` and supplies the same +token sequence to every benchmark engine. +""" + +from __future__ import annotations + +import gzip +from pathlib import Path + +MINIMUM_TIMING_TOKEN_COUNT = 5_000 + + +def _normalize(token: str) -> str: + # Java: token.trim().toLowerCase(Locale.ROOT) + return token.strip().lower() + + +def _contains_whitespace(token: str) -> bool: + return any(ch.isspace() for ch in token) + + +def read_changed_tokens(dict_gz_path: str | Path) -> list[str]: + """Return the changed-token list (token != root) in dictionary order. + + Not yet padded to the timing minimum; see :func:`build_timing_corpus`. + """ + tokens: list[str] = [] + with gzip.open(dict_gz_path, "rt", encoding="utf-8") as fh: + for line in fh: + if not line or line.isspace(): + continue + # Match the Java benchmark: only a leading marker starts a comment; + # inline markers remain part of the dictionary field. + if line.startswith("#") or line.startswith("//"): + continue + fields = line.split("\t") + if not fields: + continue + root = _normalize(fields[0]) + if not root or _contains_whitespace(root): + continue + for field in fields: + token = _normalize(field) + if not token or _contains_whitespace(token): + continue + if token != root: # changed-token workload + tokens.append(token) + return tokens + + +def build_timing_corpus( + dict_gz_path: str | Path, + minimum_token_count: int = MINIMUM_TIMING_TOKEN_COUNT, +) -> list[str]: + """Return the padded changed-token timing corpus (>= minimum_token_count).""" + changed = read_changed_tokens(dict_gz_path) + if not changed: + raise ValueError(f"No changed-token corpus tokens available in {dict_gz_path}") + if len(changed) >= minimum_token_count: + return changed + # Repeat in stable order to reach the minimum, exactly like the Java code. + out: list[str] = [] + n = len(changed) + for i in range(minimum_token_count): + out.append(changed[i % n]) + return out diff --git a/python/benchmarks/engines.py b/python/benchmarks/engines.py new file mode 100644 index 0000000..614efd9 --- /dev/null +++ b/python/benchmarks/engines.py @@ -0,0 +1,360 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Stemmer engine adapters for the benchmark. + +Every engine exposes a uniform interface: + + engine.name -> str + engine.kind -> "native-batch" | "c-batch" | "py-batch" | "py-loop" + engine.supports(code) -> bool (ISO-639-1 language code) + engine.make(code) -> callable(list[str]) -> list[str] (the batch fn) + +``kind`` records how batching is implemented: + +- ``native-batch``: one native batch call +- ``c-batch``: C extension with a list entry point +- ``py-batch``: pure-Python object with a list method +- ``py-loop``: repeated scalar calls in Python + +Engines whose package is not installed are simply reported as unavailable, so +the benchmark runs with whatever the user has. +""" + +from __future__ import annotations + +import inspect +from pathlib import Path +from typing import Callable, Optional + + +def _module_info(obj) -> dict: + """Provenance of the module that actually provides ``obj``. + + The returned metadata identifies the backing module and distinguishes + native extensions from pure-Python implementations. + + Resolves via ``type(obj).__module__`` -> ``sys.modules`` because + ``inspect.getmodule`` returns ``None`` for Cython extension instances. + """ + import sys + + name = type(obj).__module__ + mod = sys.modules.get(name) or inspect.getmodule(obj) + file = getattr(mod, "__file__", None) or "" + lower = file.lower() + compiled = ( + lower.endswith((".pyd", ".so", ".dll")) or "cpython" in lower or "abi3" in lower + ) + return { + "backing_module": name, + "backing_file": file, + "compiled_extension": compiled, + } + + +# ISO-639-1 -> Snowball algorithm name (matches the Java SnowballLanguageCase +# mapping; nb/nn both map to the single Snowball "norwegian" algorithm). +_SNOWBALL_NAMES: dict[str, str] = { + "cs": "czech", + "fa": "persian", + "pl": "polish", + "da": "danish", + "nl": "dutch", + "en": "english", + "fi": "finnish", + "fr": "french", + "de": "german", + "hu": "hungarian", + "it": "italian", + "nb": "norwegian", + "nn": "norwegian", + "pt": "portuguese", + "ru": "russian", + "es": "spanish", + "sv": "swedish", + "yi": "yiddish", +} + +BatchFn = Callable[[list[str]], list[str]] + + +class Engine: + name: str = "engine" + kind: str = "py-loop" + + def available(self) -> bool: + raise NotImplementedError + + def supports(self, code: str) -> bool: + raise NotImplementedError + + def make(self, code: str) -> BatchFn: + raise NotImplementedError + + def provenance(self, code: str) -> dict: + return { + "backing_module": self.name, + "backing_file": "", + "compiled_extension": False, + "algorithm": None, + } + + +class RadixorEngine(Engine): + name = "radixor" + kind = "native-batch" + + def __init__(self, lowercase: bool = False) -> None: + # The shared corpus is already lowercase. Disable Radixor's redundant + # normalization unless a benchmark explicitly includes that cost. + self._lowercase = lowercase + + def available(self) -> bool: + try: + import radixor # noqa: F401 + + return True + except Exception: + return False + + def supports(self, code: str) -> bool: + try: + from radixor import _LANGUAGE_ALIASES + + return code in _LANGUAGE_ALIASES + except Exception: + return False + + def make(self, code: str) -> BatchFn: + from radixor import Stemmer + + s = Stemmer(code, lowercase=self._lowercase, cache_size=0) + return s.stem_batch + + def provenance(self, code: str) -> dict: + from radixor import Stemmer + + s = Stemmer(code, lowercase=self._lowercase, cache_size=0) + info = _module_info(s._core) + info["algorithm"] = "radixor-trie" + info["lowercase"] = self._lowercase + info["cache_disabled"] = True + return info + + +class PyStemmerEngine(Engine): + name = "PyStemmer" + kind = "c-batch" + + def available(self) -> bool: + try: + import Stemmer # noqa: F401 + + return True + except Exception: + return False + + def _algorithms(self) -> set[str]: + import Stemmer + + return {a.lower() for a in Stemmer.algorithms()} + + def supports(self, code: str) -> bool: + name = _SNOWBALL_NAMES.get(code) + return bool(name) and name in self._algorithms() + + @staticmethod + def _new_stemmer(code: str): + import Stemmer + + stemmer = Stemmer.Stemmer(_SNOWBALL_NAMES[code]) + # Repeated passes would otherwise measure PyStemmer's default cache + # after the first pass. Both native engines therefore run uncached. + stemmer.maxCacheSize = 0 + return stemmer + + def make(self, code: str) -> BatchFn: + stemmer = self._new_stemmer(code) + # PyStemmer's native list entry point: one C call for the whole batch. + return stemmer.stemWords + + def provenance(self, code: str) -> dict: + stemmer = self._new_stemmer(code) + info = _module_info(stemmer) + info["algorithm"] = _SNOWBALL_NAMES[code] + info["cache_disabled"] = True + # Record whether PyStemmer resolves to an independent native extension. + try: + import snowballstemmer + + snowball_dir = str(Path(snowballstemmer.__file__).resolve().parent).lower() + except Exception: + snowball_dir = None + backing = info["backing_file"].lower() + info["independent_of_snowballstemmer"] = bool( + info["compiled_extension"] + and (snowball_dir is None or snowball_dir not in backing) + ) + return info + + +class SnowballStemmerEngine(Engine): + """Pure-Python Snowball backend. + + ``snowballstemmer.stemmer()`` delegates to PyStemmer (the C extension) when + PyStemmer is installed, which would make this engine a duplicate of + ``PyStemmer``. To retain a distinct implementation, this adapter imports + the language's pure-Python class directly from + ``snowballstemmer._stemmer``. + """ + + name = "snowballstemmer-pure" + kind = "py-batch" + + def available(self) -> bool: + try: + import snowballstemmer # noqa: F401 + + return True + except Exception: + return False + + def _load_class(self, code: str): + import importlib + + name = _SNOWBALL_NAMES.get(code) + if not name: + return None + module = importlib.import_module(f"snowballstemmer.{name}_stemmer") + class_name = name.capitalize() + "Stemmer" + return getattr(module, class_name, None) + + def supports(self, code: str) -> bool: + try: + return self._load_class(code) is not None + except Exception: + return False + + def make(self, code: str) -> BatchFn: + stemmer = self._load_class(code)() + return stemmer.stemWords # pure-Python loop over the list, internally + + def provenance(self, code: str) -> dict: + stemmer = self._load_class(code)() + info = _module_info(stemmer) + info["algorithm"] = _SNOWBALL_NAMES[code] + return info + + +class NltkPorterEngine(Engine): + name = "nltk-porter" + kind = "py-loop" + + def available(self) -> bool: + try: + from nltk.stem import PorterStemmer # noqa: F401 + + return True + except Exception: + return False + + def supports(self, code: str) -> bool: + return code == "en" # Porter is English-only + + def make(self, code: str) -> BatchFn: + from nltk.stem import PorterStemmer + + ps = PorterStemmer() + stem = ps.stem + + def batch(words: list[str]) -> list[str]: + return [stem(w) for w in words] + + return batch + + def provenance(self, code: str) -> dict: + from nltk.stem import PorterStemmer + + info = _module_info(PorterStemmer()) + info["algorithm"] = "porter" + return info + + +class CistemEngine(Engine): + """CISTEM — a fast lightweight German stemmer (German only).""" + + name = "cistem" + kind = "py-loop" + + def available(self) -> bool: + try: + from nltk.stem.cistem import Cistem # noqa: F401 + + return True + except Exception: + return False + + def supports(self, code: str) -> bool: + return code == "de" + + def make(self, code: str) -> BatchFn: + from nltk.stem.cistem import Cistem + + stem = Cistem().stem + + def batch(words: list[str]) -> list[str]: + return [stem(w) for w in words] + + return batch + + def provenance(self, code: str) -> dict: + from nltk.stem.cistem import Cistem + + info = _module_info(Cistem()) + info["algorithm"] = "cistem" + return info + + +ALL_ENGINES: list[Engine] = [ + RadixorEngine(), + PyStemmerEngine(), + SnowballStemmerEngine(), + NltkPorterEngine(), + CistemEngine(), +] + + +def available_engines(names: Optional[set[str]] = None) -> list[Engine]: + engines = [e for e in ALL_ENGINES if e.available()] + if names: + engines = [e for e in engines if e.name in names] + return engines diff --git a/python/benchmarks/requirements-bench.txt b/python/benchmarks/requirements-bench.txt new file mode 100644 index 0000000..1045292 --- /dev/null +++ b/python/benchmarks/requirements-bench.txt @@ -0,0 +1,6 @@ +# Optional comparison engines for the runtime stemming benchmark. +# radixor itself must already be built (maturin develop) in the same env. +# Any subset may be installed; the harness auto-detects what is present. +PyStemmer # Snowball as a C extension (libstemmer); real batch API +snowballstemmer # Official pure-Python Snowball (used via its pure backend) +nltk # Porter (English) pure-Python reference baseline diff --git a/python/benchmarks/run_benchmark.py b/python/benchmarks/run_benchmark.py new file mode 100644 index 0000000..be993c3 --- /dev/null +++ b/python/benchmarks/run_benchmark.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Runtime stemming benchmark for the radixor Python extension vs alternatives. + +Measures ONLY runtime stemming throughput — model construction / dictionary +compilation happens once in setup and is excluded from all timings. + +Batch sizes are swept (default 10/20/50/100) and an unconstrained descriptive +line is fitted for each engine: per_call_time(N) = intercept + slope * N. The +fit summarizes scaling across the measured sizes; timing noise can make its +intercept negative, so it must not be read as a physical overhead measurement. + +Data is the same as the Java JMH benchmarks: the changed-token corpus derived +from the bundled UniMorph gold-standard dictionaries (see corpus.py). + +Examples +-------- + python run_benchmark.py --language en + python run_benchmark.py --all-languages --engines radixor + python run_benchmark.py --language en de ru --repeats 15 --csv results.csv + python run_benchmark.py --language en --sizes 10 20 50 100 200 --json out.json +""" + +from __future__ import annotations + +import argparse +import csv as csvmod +import gc +import json +import platform +import statistics +import sys +import time +from pathlib import Path +from typing import Optional + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) # allow running as a plain script + +import corpus as corpus_mod # noqa: E402 +import engines as engines_mod # noqa: E402 + + +def _chunks(seq: list[str], n: int) -> list[list[str]]: + return [seq[i : i + n] for i in range(0, len(seq), n)] + + +def _time_sequence_ns(batch_fn, batches: list[list[str]]) -> int: + """Time one full pass over all batches (nanoseconds).""" + start = time.perf_counter_ns() + for b in batches: + batch_fn(b) + return time.perf_counter_ns() - start + + +def _linfit(xs: list[float], ys: list[float]) -> tuple[float, float]: + """Ordinary least squares: returns (intercept, slope).""" + n = len(xs) + mean_x = sum(xs) / n + mean_y = sum(ys) / n + sxx = sum((x - mean_x) ** 2 for x in xs) + sxy = sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys)) + slope = sxy / sxx if sxx else 0.0 + intercept = mean_y - slope * mean_x + return intercept, slope + + +_DIST_NAMES = { + "radixor": "radixor", + "PyStemmer": "PyStemmer", + "snowballstemmer-pure": "snowballstemmer", + "nltk-porter": "nltk", +} + + +def _engine_version(name: str) -> Optional[str]: + import importlib.metadata as md + + dist = _DIST_NAMES.get(name) + if not dist: + return None + try: + return md.version(dist) + except Exception: + return "editable" if name == "radixor" else None + + +def _processor_name() -> str: + """Return a useful CPU model name without adding a platform dependency.""" + name = platform.processor().strip() + if name: + return name + + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.is_file(): + for line in cpuinfo.read_text(encoding="utf-8", errors="replace").splitlines(): + key, separator, value = line.partition(":") + if separator and key.strip() in {"model name", "Hardware"}: + name = value.strip() + if name: + return name + + return "unknown" + + +def run(args) -> dict: + from radixor import _LANGUAGE_ALIASES + + engine_filter = set(args.engines) if args.engines else None + engines = engines_mod.available_engines(engine_filter) + if not engines: + print( + "No stemmer engines available. Install PyStemmer / snowballstemmer / nltk.", + file=sys.stderr, + ) + sys.exit(2) + + results: list[dict] = [] + strict_engine_names = ( + (engine_filter or {"radixor"}) if args.all_languages else set() + ) + failures: list[str] = [] + available_engine_names = {engine.name for engine in engines} + for missing_engine in sorted(strict_engine_names - available_engine_names): + failures.append(f"engine unavailable: {missing_engine}") + + for code in args.language: + model_id = _LANGUAGE_ALIASES.get(code, code) + if args.model_path: + dict_path = Path(args.model_path) + else: + # Corpus construction deliberately uses the canonical repository + # source. Runtime distributions contain only compiled model data. + dict_path = ( + HERE.parents[1] + / "models" + / model_id + / "src" + / "modelInput" + / "stemmer.gz" + ) + if not dict_path.is_file(): + print( + f"[{code}] canonical benchmark dictionary not found: {dict_path}", + file=sys.stderr, + ) + if args.all_languages: + failures.append(f"{code}: dictionary not found: {dict_path}") + continue + + full = corpus_mod.build_timing_corpus(dict_path) + budget = min(args.words, len(full)) if args.words > 0 else len(full) + pool = full[:budget] + print( + f"\n=== language={code} model={model_id} " + f"corpus={len(pool)} changed tokens ===" + ) + + for engine in engines: + if not engine.supports(code): + if engine.name in strict_engine_names: + failures.append(f"{code}: engine does not support {engine.name}") + continue + try: + batch_fn = engine.make(code) + except Exception as exc: # pragma: no cover - engine setup failure + print(f" [{engine.name}] setup failed: {exc}", file=sys.stderr) + if engine.name in strict_engine_names: + failures.append(f"{code}: {engine.name} setup failed: {exc}") + continue + + prov = engine.provenance(code) + results.append( + { + "language": code, + "model": model_id, + "engine": engine.name, + "kind": engine.kind, + "batch_size": "PROVENANCE", + **prov, + } + ) + print( + f" {engine.name:<16} backing={prov.get('backing_module')} " + f"compiled={prov.get('compiled_extension')} " + f"algo={prov.get('algorithm')}" + ) + + # sanity: output length must equal input length + probe = batch_fn(pool[: min(8, len(pool))]) + if len(probe) != min(8, len(pool)): + print( + f" [{engine.name}] unexpected output shape; skipping", + file=sys.stderr, + ) + if engine.name in strict_engine_names: + failures.append( + f"{code}: {engine.name} returned an unexpected output shape" + ) + continue + + per_call_best: list[float] = [] + for size in args.sizes: + batches = _chunks(pool, size) + n_calls = len(batches) + n_words = len(pool) + + # warmup + for _ in range(args.warmup): + _time_sequence_ns(batch_fn, batches) + + gc_was_enabled = gc.isenabled() + gc.disable() + try: + totals = [ + _time_sequence_ns(batch_fn, batches) + for _ in range(args.repeats) + ] + finally: + if gc_was_enabled: + gc.enable() + + med_total = statistics.median(totals) + min_total = min(totals) + # Per-word/per-call reported from the best (min) pass — the + # microbenchmark convention that suppresses OS/GC scheduling + # noise. The later OLS fit is descriptive and unconstrained. + per_word_ns = min_total / n_words + per_call_ns = min_total / n_calls + per_call_best.append(per_call_ns) + throughput = n_words / (min_total / 1e9) + + row = { + "language": code, + "model": model_id, + "engine": engine.name, + "kind": engine.kind, + "batch_size": size, + "calls": n_calls, + "words": n_words, + "repeats": args.repeats, + "median_total_ms": med_total / 1e6, + "min_total_ms": min_total / 1e6, + "per_word_ns": per_word_ns, + "per_call_us": per_call_ns / 1e3, + "throughput_words_per_s": throughput, + } + results.append(row) + print( + f" {engine.name:<16} [{engine.kind:<12}] " + f"N={size:<4} {per_word_ns:8.1f} ns/word " + f"{per_call_ns / 1e3:8.2f} us/call " + f"{throughput / 1e6:6.2f} M words/s" + ) + + # Unconstrained descriptive OLS fit across batch sizes. Keep the + # historical JSON key for report compatibility. + if len(args.sizes) >= 2: + intercept_ns, slope_ns = _linfit( + [float(s) for s in args.sizes], per_call_best + ) + results.append( + { + "language": code, + "model": model_id, + "engine": engine.name, + "kind": engine.kind, + "batch_size": "FIT", + "regie_ns_per_call": intercept_ns, + "real_ns_per_word": slope_ns, + } + ) + print( + f" {engine.name:<16} -> estimated intercept/call = " + f"{intercept_ns / 1e3:7.2f} us " + f"estimated slope = {slope_ns:7.1f} ns/word" + ) + + if args.all_languages: + expected_measurements = { + (code, engine_name, size) + for code in args.language + for engine_name in strict_engine_names + for size in args.sizes + } + actual_measurements = { + (row["language"], row["engine"], row["batch_size"]) + for row in results + if isinstance(row.get("batch_size"), int) + } + missing_measurements = sorted(expected_measurements - actual_measurements) + if missing_measurements: + failures.append(f"missing measurement rows: {missing_measurements}") + if failures: + raise RuntimeError( + "Incomplete all-language benchmark: " + "; ".join(failures) + ) + + return { + "environment": { + "platform": platform.platform(), + "processor": _processor_name(), + "python": sys.version.split()[0], + "python_impl": platform.python_implementation(), + "engine_versions": {e.name: _engine_version(e.name) for e in engines}, + }, + "parameters": { + "languages": args.language, + "sizes": args.sizes, + "words_budget": args.words, + "repeats": args.repeats, + "warmup": args.warmup, + }, + "results": results, + } + + +def main() -> None: + p = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + languages = p.add_mutually_exclusive_group() + languages.add_argument( + "--language", + "-l", + nargs="+", + default=None, + help="ISO-639-1 language code(s) or model id(s). Default: en", + ) + languages.add_argument( + "--all-languages", + action="store_true", + help="Benchmark every language alias bundled by radixor", + ) + p.add_argument( + "--sizes", + "-s", + type=int, + nargs="+", + default=[10, 20, 50, 100], + help="Batch sizes to sweep. Default: 10 20 50 100", + ) + p.add_argument( + "--words", + "-w", + type=int, + default=5000, + help="Words processed per measurement (<=0 = whole corpus). Default: 5000", + ) + p.add_argument( + "--repeats", + "-r", + type=int, + default=15, + help="Timed repeats per point (best/min reported). Default: 15", + ) + p.add_argument("--warmup", type=int, default=3, help="Warmup passes. Default: 3") + p.add_argument( + "--engines", + nargs="+", + default=None, + help="Restrict to named engines (radixor PyStemmer snowballstemmer nltk-porter)", + ) + p.add_argument( + "--model-path", + default=None, + help="Explicit gzipped dictionary path (single-language runs)", + ) + p.add_argument("--csv", default=None, help="Write per-point rows to this CSV file") + p.add_argument( + "--json", default=None, help="Write full results (incl. environment) to JSON" + ) + args = p.parse_args() + + if args.all_languages: + from radixor import _LANGUAGE_ALIASES + + args.language = sorted(_LANGUAGE_ALIASES) + elif args.language is None: + args.language = ["en"] + + report = run(args) + + if args.json: + Path(args.json).write_text(json.dumps(report, indent=2), encoding="utf-8") + print(f"\nwrote {args.json}") + if args.csv: + rows = [ + r + for r in report["results"] + if r.get("batch_size") not in ("FIT", "PROVENANCE") + ] + if rows: + with open(args.csv, "w", newline="", encoding="utf-8") as fh: + w = csvmod.DictWriter(fh, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + print(f"wrote {args.csv}") + + print("\nEnvironment:") + for k, v in report["environment"].items(): + print(f" {k}: {v}") + + +if __name__ == "__main__": + main() diff --git a/python/models-standard/LICENSE-MODEL-DATA.txt b/python/models-standard/LICENSE-MODEL-DATA.txt new file mode 100644 index 0000000..e936a23 --- /dev/null +++ b/python/models-standard/LICENSE-MODEL-DATA.txt @@ -0,0 +1,12 @@ +Radixor Standard Model Data License + +The model data in this distribution, including Radixor's protectable +selection, transformation, metadata, and packaging contributions, is licensed +under Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0): + +https://creativecommons.org/licenses/by-sa/3.0/ + +Each model's attribution, upstream provenance, and license details are recorded +in radixor_models_standard/notices//NOTICE-model-data.txt. Those +notices form part of this distribution and must be retained with redistributed +model data. diff --git a/python/models-standard/MANIFEST.in b/python/models-standard/MANIFEST.in new file mode 100644 index 0000000..3da8d75 --- /dev/null +++ b/python/models-standard/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE-MODEL-DATA.txt +include README.md +recursive-include radixor_models_standard *.json *.rxc *.txt +global-exclude stemmer.gz *.gz __pycache__ *.py[cod] diff --git a/python/models-standard/README.md b/python/models-standard/README.md new file mode 100644 index 0000000..462c026 --- /dev/null +++ b/python/models-standard/README.md @@ -0,0 +1,26 @@ +# Radixor standard models + +This pure-Python distribution supplies Radixor's 20 precompiled standard +language models. It is installed automatically by `pip install radixor`; users +normally do not import it directly. + +The catalog version is `2026.1`. Individual model versions recorded in the +generated `radixor_models_standard/manifest.json` are currently `1.0.0`. The +optional Polish PoliMorf model is intentionally not part of the standard +catalog. +The first Python model distribution is released as `1.0.0`; its version is +independent of both the catalog identity and the individual model versions. +The checked-in descriptors use `0.0.0` as a deliberate non-release placeholder. +The release workflow creates an isolated project below `build/`, injects the Git +tag version, and deterministically compiles all model resources there. + +Only gzip-compressed Radixor v7 (`.rxc`) tries are shipped. They are release +artifacts, not checked-in repository files. Canonical textual dictionaries +remain in the Radixor source repository and are not included in this wheel or +source distribution. Model data is licensed under CC BY-SA 3.0; see +`LICENSE-MODEL-DATA.txt` and the generated per-model notices. + +The checked-out directory is intentionally only a packaging skeleton and is +not directly buildable as the complete data distribution. From the repository +root, use `./gradlew pythonBuildStandardModels`; the generated project and its +wheel/sdist are written below `build/python/`. diff --git a/python/models-standard/pyproject.toml b/python/models-standard/pyproject.toml new file mode 100644 index 0000000..2b93dbe --- /dev/null +++ b/python/models-standard/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "radixor-models-standard" +version = "0.0.0" +requires-python = ">=3.9" +description = "Precompiled standard language models for Radixor" +readme = "README.md" +license = "CC-BY-SA-3.0" +license-files = ["LICENSE-MODEL-DATA.txt"] +keywords = ["stemming", "nlp", "linguistics", "model-data"] +classifiers = [ + "Programming Language :: Python :: 3", + "Topic :: Text Processing :: Linguistic", +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +include = ["radixor_models_standard*"] + +[tool.setuptools.package-data] +radixor_models_standard = ["manifest.json"] +"radixor_models_standard.models" = ["*.rxc"] +"radixor_models_standard.notices" = ["*/*.txt"] diff --git a/python/models-standard/radixor_models_standard/__init__.py b/python/models-standard/radixor_models_standard/__init__.py new file mode 100644 index 0000000..cdfcd47 --- /dev/null +++ b/python/models-standard/radixor_models_standard/__init__.py @@ -0,0 +1,37 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Installed resources for the Radixor 2026.1 standard model catalog.""" + +CATALOG_VERSION = "2026.1" +__version__ = "0.0.0" + +__all__ = ["CATALOG_VERSION"] diff --git a/python/models-standard/radixor_models_standard/models/__init__.py b/python/models-standard/radixor_models_standard/models/__init__.py new file mode 100644 index 0000000..5c26878 --- /dev/null +++ b/python/models-standard/radixor_models_standard/models/__init__.py @@ -0,0 +1,32 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Compiled model resources; not a public Python API.""" diff --git a/python/models-standard/radixor_models_standard/notices/__init__.py b/python/models-standard/radixor_models_standard/notices/__init__.py new file mode 100644 index 0000000..d04b2fb --- /dev/null +++ b/python/models-standard/radixor_models_standard/notices/__init__.py @@ -0,0 +1,32 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Per-model attribution and license notice resources.""" diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..abd6ddd --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +requires = ["maturin>=1.7,<2.0"] +build-backend = "maturin" + +[project] +name = "radixor" +version = "0.0.0" +requires-python = ">=3.9" +description = "Radixor stemmer – fastest stemming for Python, backed by Rust" +readme = "README.md" +license = "BSD-3-Clause" +license-files = ["LICENSE"] +dependencies = ["radixor-models-standard>=1.0,<2.0"] +keywords = [ + "python", + "information retrieval", + "language processing", + "morphology", + "stemming algorithms", + "stemmers", + "nlp", + "rust", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: Czech", + "Natural Language :: Danish", + "Natural Language :: Dutch", + "Natural Language :: English", + "Natural Language :: Finnish", + "Natural Language :: French", + "Natural Language :: German", + "Natural Language :: Hebrew", + "Natural Language :: Hungarian", + "Natural Language :: Italian", + "Natural Language :: Norwegian", + "Natural Language :: Persian", + "Natural Language :: Polish", + "Natural Language :: Portuguese", + "Natural Language :: Russian", + "Natural Language :: Spanish", + "Natural Language :: Swedish", + "Natural Language :: Ukrainian", + "Natural Language :: Yiddish", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Rust", + "Topic :: Internet :: WWW/HTTP :: Indexing/Search", + "Topic :: Text Processing :: Indexing", + "Topic :: Text Processing :: Linguistic", +] + +[tool.maturin] +python-source = "." +module-name = "radixor._radixor" +features = ["pyo3/extension-module"] +exclude = [ + "radixor/models/**", + "models-standard/**", + "dist/**", + "benchmarks/results*.csv", + "benchmarks/results*.json", + "**/__pycache__/**", + "**/*.pyc", + "**/*.pyo", + "**/*.pyd", +] + +[tool.ruff] +extend-exclude = ["models-standard/build", "target"] +line-length = 88 +target-version = "py39" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/python/radixor/__init__.py b/python/radixor/__init__.py new file mode 100644 index 0000000..f5f4ec8 --- /dev/null +++ b/python/radixor/__init__.py @@ -0,0 +1,398 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Python API for the Rust-backed Radixor stemmer. + +Usage:: + + from radixor import Stemmer + + s = Stemmer("en") + print(s.stem("running")) # single word + print(s.stem_batch(words)) # batch API for collections +""" + +from __future__ import annotations + +import gzip +import hashlib +import importlib.resources +import json +import re +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator, Optional + +from radixor._radixor import StemmerCore +from radixor._radixor import compile as _compile + +_LANGUAGE_ALIASES: dict[str, str] = { + # Friendly aliases -> model ID + "cs": "cs-cz-default", + "czech": "cs-cz-default", + "da": "da-dk-default", + "danish": "da-dk-default", + "de": "de-de-default", + "german": "de-de-default", + "en": "us-uk-default", + "english": "us-uk-default", + "es": "es-es-default", + "spanish": "es-es-default", + "fa": "fa-ir-default", + "persian": "fa-ir-default", + "fi": "fi-fi-default", + "finnish": "fi-fi-default", + "fr": "fr-fr-default", + "french": "fr-fr-default", + "he": "he-il-default", + "hebrew": "he-il-default", + "hu": "hu-hu-default", + "hungarian": "hu-hu-default", + "it": "it-it-default", + "italian": "it-it-default", + "nb": "nb-no-default", + "norwegian": "nb-no-default", + "nl": "nl-nl-default", + "dutch": "nl-nl-default", + "nn": "nn-no-default", + "pl": "pl-pl-unimorph", + "polish": "pl-pl-unimorph", + "pt": "pt-pt-default", + "portuguese": "pt-pt-default", + "ru": "ru-ru-default", + "russian": "ru-ru-default", + "sv": "sv-se-default", + "swedish": "sv-se-default", + "uk": "uk-ua-default", + "ukrainian": "uk-ua-default", + "yi": "yi-default", + "yiddish": "yi-default", +} + +# Right-to-left languages use FORWARD traversal; everything else BACKWARD. +# Keyed by model ID prefix (language part). +_RIGHT_TO_LEFT_MODELS: frozenset[str] = frozenset( + {"fa-ir-default", "he-il-default", "yi-default"} +) + +_STANDARD_PACKAGE = "radixor_models_standard" +_STANDARD_CATALOG_VERSION = "2026.1" +_STANDARD_DISTRIBUTION_VERSION = re.compile( + r"(?:0\.0\.0|1\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))\Z" +) +_MODEL_ID = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*\Z") +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_V7_MAGIC = b"EGTR" +_V7_VERSION = 7 + + +def _load_standard_manifest() -> dict[str, Any]: + """Load and validate the installed standard model catalog manifest.""" + try: + ref = importlib.resources.files(_STANDARD_PACKAGE).joinpath("manifest.json") + except (ModuleNotFoundError, TypeError) as exc: + raise ModuleNotFoundError( + "The standard Radixor model package is not installed. Install a compatible " + "provider with 'pip install radixor-models-standard>=1.0,<2.0', " + "or reinstall Radixor with 'pip install radixor'." + ) from exc + try: + manifest = json.loads(ref.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RuntimeError( + "The installed radixor-models-standard manifest is missing or corrupt; " + "reinstall radixor-models-standard." + ) from exc + + try: + models = manifest["models"] + format_info = manifest["format"] + if manifest["schema_version"] != 1: + raise ValueError("unsupported schema_version") + if manifest["catalog_version"] != _STANDARD_CATALOG_VERSION: + raise ValueError( + f"catalog {manifest['catalog_version']!r} is incompatible with " + f"Radixor catalog {_STANDARD_CATALOG_VERSION!r}" + ) + distribution_version = manifest["distribution_version"] + if ( + not isinstance(distribution_version, str) + or _STANDARD_DISTRIBUTION_VERSION.fullmatch(distribution_version) is None + ): + raise ValueError("incompatible distribution_version") + if format_info != {"compression": "gzip", "magic": "EGTR", "version": 7}: + raise ValueError("unsupported compiled model format") + if not isinstance(models, list) or not models: + raise ValueError("models must be a non-empty list") + seen: set[str] = set() + for model in models: + model_id = model["id"] + if ( + not isinstance(model_id, str) + or _MODEL_ID.fullmatch(model_id) is None + or model_id in seen + or model["file"] != f"models/{model_id}.rxc" + or not isinstance(model["version"], str) + or _SHA256.fullmatch(model["sha256"]) is None + ): + raise ValueError("invalid model entry") + seen.add(model_id) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError( + f"The installed radixor-models-standard manifest is incompatible or corrupt: {exc}. " + "Install radixor-models-standard>=1.0,<2.0." + ) from exc + return manifest + + +def _manifest_model(model_id: str) -> dict[str, Any]: + if not isinstance(model_id, str) or _MODEL_ID.fullmatch(model_id) is None: + raise ValueError( + f"Invalid Radixor model ID {model_id!r}; expected lowercase letters, digits, and hyphens." + ) + manifest = _load_standard_manifest() + for model in manifest["models"]: + if model["id"] == model_id: + return model + raise FileNotFoundError( + f"Model '{model_id}' is not in the standard Radixor catalog. " + "Pass a custom source path via Stemmer(path=...) or a compiled v7 path " + "via Stemmer(compiled=...)." + ) + + +def _validate_standard_model(path: Path, model: dict[str, Any]) -> None: + try: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + except OSError as exc: + raise RuntimeError( + f"Standard model '{model['id']}' cannot be read; reinstall radixor-models-standard." + ) from exc + if digest != model["sha256"]: + raise RuntimeError( + f"Standard model '{model['id']}' failed SHA-256 validation; " + "reinstall radixor-models-standard." + ) + try: + with gzip.open(path, "rb") as stream: + header = stream.read(8) + except (OSError, EOFError) as exc: + raise RuntimeError( + f"Standard model '{model['id']}' is not a valid gzip-compressed v7 resource; " + "reinstall radixor-models-standard." + ) from exc + if header[:4] != _V7_MAGIC or len(header) != 8: + raise RuntimeError( + f"Standard model '{model['id']}' does not contain the Radixor EGTR format marker; " + "reinstall radixor-models-standard." + ) + version = int.from_bytes(header[4:8], "big", signed=True) + if version != _V7_VERSION: + raise RuntimeError( + f"Standard model '{model['id']}' uses unsupported compiled format v{version}; " + f"Radixor requires v{_V7_VERSION}." + ) + + +@contextmanager +def _standard_model_path(model_id: str) -> Iterator[Path]: + """Yield a validated standard model path for synchronous native loading.""" + model = _manifest_model(model_id) + ref = ( + importlib.resources.files(_STANDARD_PACKAGE) + .joinpath("models") + .joinpath(f"{model_id}.rxc") + ) + try: + with importlib.resources.as_file(ref) as path: + if not path.is_file(): + raise FileNotFoundError + _validate_standard_model(path, model) + yield path + except FileNotFoundError as exc: + raise FileNotFoundError( + f"Standard model '{model_id}' is missing from radixor-models-standard; " + "reinstall radixor-models-standard." + ) from exc + + +def _is_backward(model_id: str) -> bool: + """Traversal direction implied by the model's language (RTL => FORWARD).""" + return model_id not in _RIGHT_TO_LEFT_MODELS + + +class Stemmer: + """Thread-safe stemmer backed by a Radixor patch-command trie. + + Standard language models are loaded from validated, precompiled v7 resources + supplied by the mandatory ``radixor-models-standard`` distribution. + + Parameters + ---------- + language: + Two-letter ISO 639-1 code (e.g. ``"en"``) or a full model ID + (e.g. ``"us-uk-default"``). Ignored when ``path`` is given. + path: + Explicit path to either a gzipped source dictionary or a compiled + ``.rxc`` trie (Java-interoperable v7 format); the format is + auto-detected. Takes precedence over ``language``. + compiled: + Alias for ``path`` intended for compiled ``.rxc`` files (see + :func:`compile`). For compiled input, ``backward`` / ``store_original`` + are baked into the file and ignored. + backward: + Traversal direction override. When ``None`` (default) it is derived + from the language (BACKWARD, except right-to-left fa/he/yi which use + FORWARD). Only consulted for ``path``-based construction if given. + store_original: + When ``True`` (default) each canonical stem maps to the no-op patch, + so the stem itself is recognised. + lowercase: + When ``True`` (default) lookups lowercase the input word. Set to + ``False`` when you guarantee the input is already lowercased (skips the + per-lookup normalization; the model's keys are always lowercase). + cache_size: + Maximum entries in the bounded result cache (default ``10_000``, + matching PyStemmer). Set to ``0`` to disable caching. Cached results are + shared by :meth:`stem`, :meth:`stemWord`, :meth:`stem_batch`, and + :meth:`stemWords`; ``stem_all`` methods are not cached. + """ + + def __init__( + self, + language: Optional[str] = None, + *, + path: Optional[str] = None, + compiled: Optional[str] = None, + backward: Optional[bool] = None, + store_original: bool = True, + lowercase: bool = True, + cache_size: int = 10_000, + ) -> None: + source = path if path is not None else compiled + if source is not None: + model_path = source + is_backward = True if backward is None else backward + elif language is not None: + model_id = _LANGUAGE_ALIASES.get(language, language) + is_backward = _is_backward(model_id) if backward is None else backward + with _standard_model_path(model_id) as model_path: + self._core = StemmerCore( + str(model_path), is_backward, store_original, lowercase, cache_size + ) + return + else: + raise ValueError("Provide 'language', 'path', or 'compiled'.") + self._core = StemmerCore( + model_path, is_backward, store_original, lowercase, cache_size + ) + + def stem(self, word: str) -> Optional[str]: + """Return a stem, or ``None`` when no patch command applies.""" + return self._core.stem(word) + + def stem_batch(self, words: list[str]) -> list[Optional[str]]: + """Stem many words in one call. + + Preferred over calling :meth:`stem` in a loop: the Python→Rust bridge + overhead is amortised across the whole batch, making this significantly + faster for large word lists. + + Returns a list of the same length; entries are ``None`` when the + compiled trie finds no applicable patch command. + """ + return self._core.stem_batch(words) + + def stemWord(self, word: str) -> str: + """Return a stem using PyStemmer-compatible fallback semantics. + + If no patch command can be found, return *word* unchanged. Use + :meth:`stem` when a missing result must remain distinguishable as + ``None``. + """ + return self._core.stemWord(word) + + def stemWords(self, words: list[str]) -> list[str]: + """Stem words using PyStemmer-compatible fallback semantics. + + The returned list has the same length and order as *words*; each word + without a matching patch command is returned unchanged. + """ + return self._core.stemWords(words) + + def stem_all(self, word: str) -> list[str]: + """Return all stems for *word* ordered by descending frequency.""" + return self._core.stem_all(word) + + def stem_all_batch(self, words: list[str]) -> list[list[str]]: + """Return all stems for each word in *words* as a list of lists.""" + return self._core.stem_all_batch(words) + + +def compile( + source: str, + out_path: str, + *, + language: Optional[str] = None, + backward: Optional[bool] = None, + store_original: bool = True, + lowercase: bool = True, +) -> None: + """Compile a textual source dictionary into a Java-interoperable compiled + trie file (v7 format) that :class:`Stemmer` can load instantly. + + Parameters + ---------- + source: + Path to a gzipped (or plain) TSV source dictionary. + out_path: + Destination compiled file (conventionally ``*.rxc``). + language: + Optional language code/model ID used only to derive ``backward`` when + it is not given (right-to-left fa/he/yi compile FORWARD). + backward: + Traversal direction. When ``None`` it is derived from ``language`` if + provided, otherwise defaults to BACKWARD. + store_original, lowercase: + Same meaning as :class:`Stemmer`; baked into the compiled file. + + The resulting file is byte-compatible (inner stream) with the Radixor Java + ``StemmerPatchTrieBinaryIO`` v7 format, so Java and Python can share it. + """ + if backward is None: + if language is not None: + backward = _is_backward(_LANGUAGE_ALIASES.get(language, language)) + else: + backward = True + _compile(source, out_path, backward, store_original, lowercase) + + +__all__ = ["Stemmer", "compile"] diff --git a/python/radixor/py.typed b/python/radixor/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python/scripts/assemble_release.py b/python/scripts/assemble_release.py new file mode 100755 index 0000000..7298508 --- /dev/null +++ b/python/scripts/assemble_release.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Validate and assemble the exact files allowed in a Python GitHub Release.""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import shutil +from pathlib import Path + +from verify_distributions import ( + _verify_main_sdist, + _verify_main_wheel, + _verify_standard_sdist, + _verify_standard_wheel, +) + +REPOSITORY = Path(__file__).resolve().parents[2] +BUILD_ROOT = (REPOSITORY / "build").resolve() +VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z") + + +def _release_files(root: Path) -> list[Path]: + files = sorted( + path + for path in root.rglob("*") + if path.is_file() + and (path.name.endswith(".whl") or path.name.endswith(".tar.gz")) + ) + names = [path.name for path in files] + if len(names) != len(set(names)): + raise ValueError(f"Duplicate release filenames: {names}") + return files + + +def _require_one(files: list[Path], predicate, description: str) -> Path: + matches = [path for path in files if predicate(path.name)] + if len(matches) != 1: + raise ValueError( + f"Expected one {description}, found {[path.name for path in matches]}" + ) + return matches[0] + + +def _validate_native(files: list[Path], version: str) -> None: + prefix = f"radixor-{version}-cp39-abi3-" + wheels = [path for path in files if path.suffix == ".whl"] + sdist = _require_one( + files, lambda name: name == f"radixor-{version}.tar.gz", "native sdist" + ) + if len(wheels) != 4: + raise ValueError( + f"Expected four native wheels, found {[path.name for path in wheels]}" + ) + expected = { + "linux-x86_64": lambda name: name.startswith(prefix) + and "manylinux" in name + and name.endswith("x86_64.whl"), + "linux-aarch64": lambda name: name.startswith(prefix) + and "manylinux" in name + and name.endswith("aarch64.whl"), + "macos-universal2": lambda name: name.startswith(prefix) + and "macosx" in name + and name.endswith("universal2.whl"), + "windows-x86_64": lambda name: name == f"{prefix}win_amd64.whl", + } + for description, predicate in expected.items(): + _require_one(wheels, predicate, description) + for wheel in wheels: + _verify_main_wheel(wheel, version) + _verify_main_sdist(sdist, version) + + +def _validate_models(files: list[Path], version: str) -> None: + wheel = _require_one( + files, + lambda name: name == f"radixor_models_standard-{version}-py3-none-any.whl", + "standard-model wheel", + ) + sdist = _require_one( + files, + lambda name: name == f"radixor_models_standard-{version}.tar.gz", + "standard-model sdist", + ) + if len(files) != 2: + raise ValueError( + f"Unexpected standard-model release files: {[path.name for path in files]}" + ) + _verify_standard_wheel(wheel, version) + _verify_standard_sdist(sdist, version) + + +def _output_directory(path: Path) -> Path: + output = path.resolve() + try: + output.relative_to(BUILD_ROOT) + except ValueError as exc: + raise ValueError(f"Release output must be below {BUILD_ROOT}") from exc + if output == BUILD_ROOT: + raise ValueError("Release output cannot be the build root") + if output.exists(): + shutil.rmtree(output) + output.mkdir(parents=True) + return output + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("distribution", choices=("native", "models-standard")) + parser.add_argument("version") + parser.add_argument("artifacts", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + + if VERSION.fullmatch(args.version) is None: + raise SystemExit(f"Invalid stable release version: {args.version!r}") + files = _release_files(args.artifacts.resolve()) + if args.distribution == "native": + _validate_native(files, args.version) + else: + _validate_models(files, args.version) + + output = _output_directory(args.output) + for source in files: + shutil.copy2(source, output / source.name) + checksums = [ + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}" + for path in sorted(output.iterdir()) + if path.is_file() + ] + (output / "SHA256SUMS").write_text( + "\n".join(checksums) + "\n", encoding="utf-8", newline="\n" + ) + print(f"assembled {len(files)} release artifacts in {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/scripts/build_standard_distribution.py b/python/scripts/build_standard_distribution.py new file mode 100644 index 0000000..01ae68f --- /dev/null +++ b/python/scripts/build_standard_distribution.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Build the standard-model wheel and sdist through its declared backend.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import tempfile +from pathlib import Path + +EXPECTED_MODEL_COUNT = 20 + + +def _ignore_build_artifacts(_directory: str, names: list[str]) -> set[str]: + """Exclude local build state from the isolated distribution source tree.""" + + ignored = { + name + for name in names + if name in {"build", "dist", "__pycache__"} + or name.endswith((".egg-info", ".pyc", ".pyo")) + } + return ignored + + +def _validate_generated_project(project: Path) -> None: + """Reject an ungenerated or incomplete standard-model project.""" + + package = project / "radixor_models_standard" + manifest_path = package / "manifest.json" + if not manifest_path.is_file(): + raise ValueError( + "standard-model manifest is missing; run build_standard_models.py first" + ) + + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + model_ids = { + model["id"] for model in manifest["models"] if isinstance(model, dict) + } + except (json.JSONDecodeError, KeyError, TypeError) as exc: + raise ValueError("standard-model manifest is invalid") from exc + + compiled = {path.stem for path in (package / "models").glob("*.rxc")} + notices = { + path.parent.name + for path in (package / "notices").glob("*/NOTICE-model-data.txt") + } + if len(model_ids) != EXPECTED_MODEL_COUNT: + raise ValueError( + f"expected {EXPECTED_MODEL_COUNT} standard models, found {len(model_ids)}" + ) + if compiled != model_ids: + raise ValueError("compiled standard models do not match the manifest") + if notices != model_ids: + raise ValueError("standard-model notices do not match the manifest") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project", required=True, type=Path) + parser.add_argument("--outdir", required=True, type=Path) + args = parser.parse_args() + + try: + from setuptools.build_meta import build_sdist, build_wheel + except ImportError as exc: + raise SystemExit( + "setuptools>=77 is required to build radixor-models-standard" + ) from exc + + source_project = args.project.resolve() + try: + _validate_generated_project(source_project) + except ValueError as exc: + parser.error(str(exc)) + output = args.outdir.resolve() + output.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="radixor-models-build-") as temporary: + project = Path(temporary, "project") + shutil.copytree( + source_project, + project, + ignore=_ignore_build_artifacts, + ) + os.chdir(project) + wheel = build_wheel(str(output)) + sdist = build_sdist(str(output)) + + print(f"built {wheel} and {sdist}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/scripts/build_standard_models.py b/python/scripts/build_standard_models.py new file mode 100644 index 0000000..616e767 --- /dev/null +++ b/python/scripts/build_standard_models.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Regenerate the pure-Python standard model package deterministically. + +The canonical build topology selects ``default`` models. Source dictionaries +are compiler inputs only and are never copied into either Python distribution. +Run this script with a built Radixor extension importable by the selected +Python interpreter. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import sys +import tempfile +from pathlib import Path +from typing import Any + +REPOSITORY = Path(__file__).resolve().parents[2] +PYTHON_ROOT = REPOSITORY / "python" +BUILD_ROOT = REPOSITORY / "build" +SOURCE_PROJECT = PYTHON_ROOT / "models-standard" +TOPOLOGY = REPOSITORY / "models" / "model-projects.properties" +CATALOG_VERSION = REPOSITORY / "models" / "catalog-version.txt" +EXPECTED_FORMAT = {"compression": "gzip", "magic": "EGTR", "version": 7} +VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z") +METADATA_FIELDS = ( + "sourceName", + "sourceVersion", + "sourceRevision", + "sourceProject", + "sourceRepository", + "sourceDataset", + "sourceRevisionStatus", + "sourceLicense", + "sourceLicenseUri", + "sourceAttribution", + "sourceVerificationDate", + "transformationsSummary", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _default_models() -> list[str]: + entries: dict[str, str] = {} + for raw_line in TOPOLOGY.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + model_id, separator, membership = line.partition("=") + if not separator or not model_id or membership not in {"default", "optional"}: + raise ValueError(f"Invalid model topology line: {raw_line!r}") + entries[model_id] = membership + model_ids = sorted( + model_id for model_id, membership in entries.items() if membership == "default" + ) + if len(model_ids) != 20 or "pl-pl-polimorf" in model_ids: + raise ValueError( + "Standard Python catalog must contain 20 defaults and exclude pl-pl-polimorf" + ) + return model_ids + + +def _gradle_metadata(path: Path) -> dict[str, str]: + text = path.read_text(encoding="utf-8") + result: dict[str, str] = {} + for field in METADATA_FIELDS: + match = re.search(rf"^\s*{field}\s*=\s*'([^']*)'\s*$", text, re.MULTILINE) + if match is None: + raise ValueError(f"Missing {field} in {path}") + result[field] = match.group(1) + return result + + +def _replace_once(path: Path, pattern: str, replacement: str) -> None: + text = path.read_text(encoding="utf-8") + updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE) + if count != 1: + raise ValueError(f"Expected exactly one version field in {path}") + path.write_text(updated, encoding="utf-8", newline="\n") + + +def _stage_project(output: Path, distribution_version: str) -> Path: + project = output.resolve() + try: + project.relative_to(BUILD_ROOT.resolve()) + except ValueError as exc: + raise ValueError(f"Generated model project must be below {BUILD_ROOT}") from exc + if project == BUILD_ROOT.resolve(): + raise ValueError("Generated model project cannot be the build root") + if project.exists(): + shutil.rmtree(project) + project.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree( + SOURCE_PROJECT, + project, + ignore=shutil.ignore_patterns( + "build", + "dist", + "*.egg-info", + "__pycache__", + "*.pyc", + "manifest.json", + "*.rxc", + "NOTICE-model-data.txt", + ), + ) + _replace_once( + project / "pyproject.toml", + r'^version = "0\.0\.0"$', + f'version = "{distribution_version}"', + ) + _replace_once( + project / "radixor_models_standard" / "__init__.py", + r'^__version__ = "0\.0\.0"$', + f'__version__ = "{distribution_version}"', + ) + return project + + +def _model_manifest( + model_id: str, + compile_model: Any, + package_root: Path, + reproducibility_directory: Path, +) -> dict[str, Any]: + project = REPOSITORY / "models" / model_id + source = project / "src" / "modelInput" / "stemmer.gz" + notice = project / "src" / "modelInput" / "NOTICE-model-data.txt" + version = (project / "model-version.txt").read_text(encoding="utf-8").strip() + if not source.is_file() or not notice.is_file() or not version: + raise FileNotFoundError(f"Incomplete canonical model project: {project}") + + destination = package_root / "models" / f"{model_id}.rxc" + destination.parent.mkdir(parents=True, exist_ok=True) + compile_model(str(source), str(destination), language=model_id) + reproduction = reproducibility_directory / destination.name + compile_model(str(source), str(reproduction), language=model_id) + if destination.read_bytes() != reproduction.read_bytes(): + raise ValueError(f"Non-deterministic compiled model output for {model_id}") + + notice_destination = package_root / "notices" / model_id / notice.name + notice_destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(notice, notice_destination) + + metadata = _gradle_metadata(project / "build.gradle") + if metadata["sourceLicense"] != "CC-BY-SA-3.0": + raise ValueError(f"Standard model {model_id} must be CC-BY-SA-3.0") + return { + "file": f"models/{model_id}.rxc", + "id": model_id, + "notice": f"notices/{model_id}/{notice.name}", + "provenance": { + "attribution": metadata["sourceAttribution"], + "dataset": metadata["sourceDataset"], + "license": metadata["sourceLicense"], + "license_uri": metadata["sourceLicenseUri"], + "repository": metadata["sourceRepository"], + "revision": metadata["sourceRevision"], + "revision_status": metadata["sourceRevisionStatus"], + "source_name": metadata["sourceName"], + "source_project": metadata["sourceProject"], + "source_version": metadata["sourceVersion"], + "transformations": metadata["transformationsSummary"], + "verification_date": metadata["sourceVerificationDate"], + }, + "sha256": _sha256(destination), + "source": { + "path": f"models/{model_id}/src/modelInput/stemmer.gz", + "sha256": _sha256(source), + }, + "version": version, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project", required=True, type=Path) + parser.add_argument("--distribution-version", required=True) + args = parser.parse_args() + if VERSION.fullmatch(args.distribution_version) is None: + raise SystemExit( + f"Invalid standard-model distribution version: {args.distribution_version!r}" + ) + + try: + from radixor import compile as compile_model + except ImportError as exc: + print( + "error: build the Radixor extension first (for example, " + "maturin develop --release) and rerun this script", + file=sys.stderr, + ) + print(f"detail: {exc}", file=sys.stderr) + return 2 + + project = _stage_project(args.project, args.distribution_version) + package_root = project / "radixor_models_standard" + model_ids = _default_models() + expected_files = {f"{model_id}.rxc" for model_id in model_ids} + models_directory = package_root / "models" + models_directory.mkdir(parents=True, exist_ok=True) + for stale in models_directory.glob("*.rxc"): + if stale.name not in expected_files: + stale.unlink() + notices_directory = package_root / "notices" + notices_directory.mkdir(parents=True, exist_ok=True) + for stale in notices_directory.iterdir(): + if stale.is_dir() and stale.name not in model_ids: + shutil.rmtree(stale) + + reproducibility_root = BUILD_ROOT / "python" / "tmp" + reproducibility_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix="radixor-model-reproducibility-", dir=reproducibility_root + ) as temporary: + models = [ + _model_manifest( + model_id, + compile_model, + package_root, + Path(temporary), + ) + for model_id in model_ids + ] + manifest = { + "catalog_version": CATALOG_VERSION.read_text(encoding="utf-8").strip(), + "distribution_version": args.distribution_version, + "format": EXPECTED_FORMAT, + "models": models, + "schema_version": 1, + "topology": "models/model-projects.properties", + } + manifest_path = package_root / "manifest.json" + with manifest_path.open("w", encoding="utf-8", newline="\n") as stream: + stream.write( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ) + print(f"generated {len(models)} standard compiled models in {package_root}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/scripts/prepare_release_tree.py b/python/scripts/prepare_release_tree.py new file mode 100755 index 0000000..35d4c59 --- /dev/null +++ b/python/scripts/prepare_release_tree.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Create an isolated, tag-versioned Python release source tree.""" + +from __future__ import annotations + +import argparse +import re +import shutil +from pathlib import Path + +REPOSITORY = Path(__file__).resolve().parents[2] +PYTHON_ROOT = REPOSITORY / "python" +BUILD_ROOT = REPOSITORY / "build" +VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z") + + +def _replace_once(path: Path, pattern: str, replacement: str) -> None: + text = path.read_text(encoding="utf-8") + updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE) + if count != 1: + raise ValueError(f"Expected exactly one version field in {path}") + path.write_text(updated, encoding="utf-8", newline="\n") + + +def _prepare_output(path: Path) -> Path: + output = path.resolve() + try: + output.relative_to(BUILD_ROOT.resolve()) + except ValueError as exc: + raise ValueError(f"Release staging output must be below {BUILD_ROOT}") from exc + if output == BUILD_ROOT.resolve(): + raise ValueError("Release staging output cannot be the build root") + if output.exists(): + shutil.rmtree(output) + output.parent.mkdir(parents=True, exist_ok=True) + return output + + +def _copy_native(output: Path, version: str) -> None: + shutil.copytree( + PYTHON_ROOT, + output, + ignore=shutil.ignore_patterns( + ".gitignore", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + "*.pyc", + "_radixor*.dll", + "_radixor*.dylib", + "_radixor*.pyd", + "_radixor*.so", + "_native*.dll", + "_native*.dylib", + "_native*.pyd", + "_native*.so", + "benchmarks", + "dist", + "models", + "models-standard", + "target", + "tests", + ), + ) + _replace_once( + output / "pyproject.toml", + r'^version = "0\.0\.0"$', + f'version = "{version}"', + ) + _replace_once( + output / "Cargo.toml", + r'^version = "0\.0\.0"$', + f'version = "{version}"', + ) + lock = output / "Cargo.lock" + lock_text = lock.read_text(encoding="utf-8") + pattern = r'(\[\[package\]\]\nname = "radixor"\n)version = "0\.0\.0"' + lock_text, count = re.subn(pattern, rf'\g<1>version = "{version}"', lock_text) + if count != 1: + raise ValueError("Expected exactly one radixor package entry in Cargo.lock") + lock.write_text(lock_text, encoding="utf-8", newline="\n") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("distribution", choices=("native",)) + parser.add_argument("version") + parser.add_argument("output", type=Path) + args = parser.parse_args() + + if VERSION.fullmatch(args.version) is None: + raise SystemExit(f"Invalid stable release version: {args.version!r}") + output = _prepare_output(args.output) + _copy_native(output, args.version) + print(f"prepared {args.distribution} {args.version} in {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/scripts/update_simple_index.py b/python/scripts/update_simple_index.py new file mode 100755 index 0000000..8b113a9 --- /dev/null +++ b/python/scripts/update_simple_index.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Update Radixor's static PEP 503 index with verified GitHub Release assets.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import re +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import quote + +PACKAGES = ("radixor", "radixor-models-standard") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z") + + +class _AnchorParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.anchors: dict[str, str] = {} + self._href: str | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + self._href = dict(attrs).get("href") + + def handle_data(self, data: str) -> None: + if self._href is not None and data.strip(): + self.anchors[data.strip()] = self._href + + def handle_endtag(self, tag: str) -> None: + if tag == "a": + self._href = None + + +def _existing_links(path: Path, expected_prefix: str) -> dict[str, str]: + if not path.exists(): + return {} + parser = _AnchorParser() + parser.feed(path.read_text(encoding="utf-8")) + for filename, href in parser.anchors.items(): + base, separator, digest = href.rpartition("#sha256=") + if ( + not separator + or not base.startswith(expected_prefix) + or SHA256.fullmatch(digest) is None + or base.rsplit("/", 1)[-1] != quote(filename, safe="._-") + ): + raise ValueError(f"Existing package-index link is unsafe: {href!r}") + return parser.anchors + + +def _render_root(root: Path) -> None: + links = "\n".join( + f' {html.escape(package)}
' + for package in PACKAGES + ) + (root / "index.html").write_text( + '\nRadixor Python packages\n' + f"\n{links}\n\n", + encoding="utf-8", + newline="\n", + ) + + +def _render_project(path: Path, package: str, links: dict[str, str]) -> None: + anchors = "\n".join( + f' {html.escape(filename)}
' + for filename, href in sorted(links.items()) + ) + path.write_text( + '\nLinks for {html.escape(package)}\n' + f"\n{anchors}\n\n", + encoding="utf-8", + newline="\n", + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True, type=Path) + parser.add_argument("--repository", required=True) + parser.add_argument("--package", required=True, choices=PACKAGES) + parser.add_argument("--version", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--artifacts", required=True, type=Path) + args = parser.parse_args() + + if REPOSITORY.fullmatch(args.repository) is None: + raise SystemExit("Invalid GitHub repository identity") + if VERSION.fullmatch(args.version) is None: + raise SystemExit("Invalid stable release version") + expected_tag = ( + f"python@{args.version}" + if args.package == "radixor" + else f"python-models-standard@{args.version}" + ) + if args.tag != expected_tag: + raise SystemExit(f"Tag {args.tag!r} does not match {expected_tag!r}") + + root = args.root.resolve() + if root.name != "simple" or root.parent.name != "python": + raise SystemExit("The package index must end in python/simple") + root.mkdir(parents=True, exist_ok=True) + project = root / args.package + project.mkdir(exist_ok=True) + page = project / "index.html" + prefix = f"https://github.com/{args.repository}/releases/download/" + links = _existing_links(page, prefix) + release_prefix = f"{prefix}{quote(args.tag, safe='@._-')}/" + artifacts = sorted( + path + for path in args.artifacts.iterdir() + if path.is_file() + and (path.name.endswith(".whl") or path.name.endswith(".tar.gz")) + ) + if not artifacts: + raise SystemExit("No package artifacts were provided") + for artifact in artifacts: + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + href = f"{release_prefix}{quote(artifact.name, safe='._-')}#sha256={digest}" + previous = links.get(artifact.name) + if previous is not None and previous != href: + raise ValueError( + f"Refusing to replace existing index entry {artifact.name}" + ) + links[artifact.name] = href + + _render_root(root) + _render_project(page, args.package, links) + print(f"indexed {len(artifacts)} artifacts for {args.package} {args.version}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/scripts/verify_distributions.py b/python/scripts/verify_distributions.py new file mode 100644 index 0000000..a1cbef0 --- /dev/null +++ b/python/scripts/verify_distributions.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Verify Radixor's native and standard-model release archives.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import stat +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from email.parser import BytesParser +from pathlib import Path, PurePosixPath + +EXPECTED_DEFAULT_COUNT = 20 +EXPECTED_DEPENDENCY = "radixor-models-standard>=1.0,<2.0" +REPOSITORY = Path(__file__).resolve().parents[2] +EXPECTED_NATIVE_LICENSE = REPOSITORY.joinpath("LICENSE").read_bytes() +EXPECTED_MODEL_IDS = { + "cs-cz-default", + "da-dk-default", + "de-de-default", + "es-es-default", + "fa-ir-default", + "fi-fi-default", + "fr-fr-default", + "he-il-default", + "hu-hu-default", + "it-it-default", + "nb-no-default", + "nl-nl-default", + "nn-no-default", + "pl-pl-unimorph", + "pt-pt-default", + "ru-ru-default", + "sv-se-default", + "uk-ua-default", + "us-uk-default", + "yi-default", +} +STANDARD_NAME = "radixor-models-standard" +STANDARD_VERSION = "0.0.0" +MAIN_NAME = "radixor" +MAIN_VERSION = "0.0.0" + + +def _one(directory: Path, pattern: str) -> Path: + matches = sorted(directory.glob(pattern)) + if len(matches) != 1: + raise ValueError( + f"Expected exactly one {pattern} in {directory}, found {len(matches)}" + ) + return matches[0] + + +def _assert_no_repository_temp_gzip() -> None: + offenders = sorted( + path.name for path in REPOSITORY.glob("tmp*.gz") if path.is_file() + ) + if offenders: + raise ValueError( + "Repository root contains unmanaged Python temporary gzip files; " + f"remove them and fix the producing workflow: {offenders}" + ) + + +def _wheel_metadata(archive: zipfile.ZipFile) -> object: + names = [ + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ] + if len(names) != 1: + raise ValueError("Wheel must contain exactly one METADATA file") + return BytesParser().parsebytes(archive.read(names[0])) + + +def _normalized_member_name(name: str, archive_name: str) -> str: + if not name or "\\" in name or name.startswith("/"): + raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}") + stripped = name[:-1] if name.endswith("/") else name + parts = stripped.split("/") + if not stripped or any(part in {"", ".", ".."} for part in parts): + raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}") + return "/".join(parts) + + +def _validate_zip_members(archive: zipfile.ZipFile, archive_name: str) -> list[str]: + names: list[str] = [] + seen: set[str] = set() + for info in archive.infolist(): + name = _normalized_member_name(info.filename, archive_name) + if name in seen: + raise ValueError(f"{archive_name} contains duplicate member: {name}") + seen.add(name) + mode = (info.external_attr >> 16) & 0xFFFF + if stat.S_ISLNK(mode): + raise ValueError(f"{archive_name} contains symlink-like ZIP member: {name}") + names.append(name) + return names + + +def _validate_tar_members(archive: tarfile.TarFile, archive_name: str) -> list[str]: + names: list[str] = [] + seen: set[str] = set() + for member in archive.getmembers(): + name = _normalized_member_name(member.name, archive_name) + if name in seen: + raise ValueError(f"{archive_name} contains duplicate member: {name}") + seen.add(name) + if not (member.isfile() or member.isdir()): + raise ValueError(f"{archive_name} contains unsafe special member: {name}") + names.append(name) + return names + + +def _assert_distribution_metadata( + metadata: object, name: str, version: str, archive_name: str +) -> None: + if metadata.get("Name") != name or metadata.get("Version") != version: + raise ValueError( + f"{archive_name} has unexpected distribution identity: " + f"{metadata.get('Name')} {metadata.get('Version')}" + ) + + +def _tar_file_bytes(archive: tarfile.TarFile, name: str) -> bytes: + member = archive.getmember(name) + stream = archive.extractfile(member) + if stream is None: + raise ValueError(f"Cannot read archive member: {name}") + return stream.read() + + +def _validate_standard_manifest( + manifest: object, expected_version: str = STANDARD_VERSION +) -> list[dict]: + if not isinstance(manifest, dict) or set(manifest) != { + "catalog_version", + "distribution_version", + "format", + "models", + "schema_version", + "topology", + }: + raise ValueError("Standard model manifest has an invalid top-level schema") + if ( + manifest["schema_version"] != 1 + or manifest["catalog_version"] != "2026.1" + or manifest["distribution_version"] != expected_version + or manifest["topology"] != "models/model-projects.properties" + or manifest["format"] != {"compression": "gzip", "magic": "EGTR", "version": 7} + or not isinstance(manifest["models"], list) + ): + raise ValueError("Standard model manifest has incompatible catalog metadata") + models = manifest["models"] + if { + model.get("id") for model in models if isinstance(model, dict) + } != EXPECTED_MODEL_IDS: + raise ValueError( + "Standard model manifest does not match the default model topology" + ) + if len(models) != len(EXPECTED_MODEL_IDS): + raise ValueError("Standard model manifest contains duplicate model entries") + provenance_keys = { + "attribution", + "dataset", + "license", + "license_uri", + "repository", + "revision", + "revision_status", + "source_name", + "source_project", + "source_version", + "transformations", + "verification_date", + } + for model in models: + if set(model) != { + "file", + "id", + "notice", + "provenance", + "sha256", + "source", + "version", + }: + raise ValueError("Standard model manifest contains an invalid model entry") + model_id = model["id"] + if ( + model["file"] != f"models/{model_id}.rxc" + or model["notice"] != f"notices/{model_id}/NOTICE-model-data.txt" + or model["version"] != "1.0.0" + or not isinstance(model["sha256"], str) + or len(model["sha256"]) != 64 + or set(model["provenance"]) != provenance_keys + or model["provenance"]["license"] != "CC-BY-SA-3.0" + or set(model["source"]) != {"path", "sha256"} + or model["source"]["path"] != f"models/{model_id}/src/modelInput/stemmer.gz" + ): + raise ValueError(f"Standard model manifest entry is invalid: {model_id}") + if not all(isinstance(value, str) for value in model["provenance"].values()): + raise ValueError(f"Standard model provenance is invalid: {model_id}") + return models + + +def _assert_no_source_dictionaries(names: list[str], archive_name: str) -> None: + offenders = [ + name + for name in names + if name.endswith("stemmer.gz") or "/radixor/models/" in name + ] + if offenders: + raise ValueError( + f"{archive_name} contains forbidden source dictionaries: {offenders}" + ) + + +def _assert_no_standard_model_payload(names: list[str], archive_name: str) -> None: + offenders = [] + for name in names: + parts = PurePosixPath(name).parts + if ( + name.endswith(".rxc") + or "models-standard" in parts + or any(part.startswith("radixor_models_standard") for part in parts) + ): + offenders.append(name) + if offenders: + raise ValueError( + f"{archive_name} contains standard-model payload owned by " + f"radixor-models-standard: {offenders}" + ) + + +def _assert_no_local_build_outputs(names: list[str], archive_name: str) -> None: + offenders = [] + for name in names: + path = PurePosixPath(name) + parts = path.parts + is_benchmark_result = ( + "benchmarks" in parts + and path.name.startswith("results") + and path.suffix in {".csv", ".json"} + ) + if "dist" in parts or is_benchmark_result: + offenders.append(name) + if offenders: + raise ValueError( + f"{archive_name} contains local distribution or benchmark outputs: {offenders}" + ) + + +def _assert_no_python_cache( + names: list[str], archive_name: str, *, allow_native_pyd: bool = False +) -> None: + offenders = [] + for name in names: + path = PurePosixPath(name) + is_native_module = ( + allow_native_pyd + and path.suffix == ".pyd" + and path.name.startswith("_radixor") + ) + if ( + "__pycache__" in path.parts + or path.suffix in {".pyc", ".pyo"} + or (path.suffix == ".pyd" and not is_native_module) + ): + offenders.append(name) + if offenders: + raise ValueError(f"{archive_name} contains Python cache artifacts: {offenders}") + + +def _assert_main_dependency(metadata, archive_name: str) -> None: + requirements = metadata.get_all("Requires-Dist", []) + normalized = {requirement.replace(" ", "") for requirement in requirements} + if EXPECTED_DEPENDENCY not in normalized: + raise ValueError( + f"Missing compatible standard-model dependency in {archive_name}: " + f"{requirements}" + ) + + +def _verify_main_wheel(path: Path, expected_version: str = MAIN_VERSION) -> None: + with zipfile.ZipFile(path) as archive: + names = _validate_zip_members(archive, path.name) + _assert_no_source_dictionaries(names, path.name) + _assert_no_standard_model_payload(names, path.name) + _assert_no_python_cache(names, path.name, allow_native_pyd=True) + metadata = _wheel_metadata(archive) + _assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name) + if metadata.get("License-Expression") != "BSD-3-Clause": + raise ValueError( + f"{path.name} must declare License-Expression: BSD-3-Clause" + ) + license_names = [ + name for name in names if name.endswith(".dist-info/licenses/LICENSE") + ] + if ( + len(license_names) != 1 + or archive.read(license_names[0]) != EXPECTED_NATIVE_LICENSE + ): + raise ValueError( + f"{path.name} does not contain the full repository BSD-3-Clause LICENSE" + ) + _assert_main_dependency(metadata, path.name) + + +def _verify_main_sdist(path: Path, expected_version: str = MAIN_VERSION) -> None: + with tarfile.open(path, "r:gz") as archive: + names = _validate_tar_members(archive, path.name) + _assert_no_source_dictionaries(names, path.name) + _assert_no_standard_model_payload(names, path.name) + _assert_no_local_build_outputs(names, path.name) + _assert_no_python_cache(names, path.name) + license_members = [ + member + for member in archive.getmembers() + if PurePosixPath(member.name).name == "LICENSE" and member.isfile() + ] + if len(license_members) != 1: + raise ValueError( + f"{path.name} must contain exactly one BSD-3-Clause LICENSE" + ) + license_stream = archive.extractfile(license_members[0]) + if license_stream is None or license_stream.read() != EXPECTED_NATIVE_LICENSE: + raise ValueError( + f"{path.name} does not contain the full repository BSD-3-Clause LICENSE" + ) + root = f"radixor-{expected_version}" + metadata = BytesParser().parsebytes( + _tar_file_bytes(archive, f"{root}/PKG-INFO") + ) + _assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name) + _assert_main_dependency(metadata, path.name) + + +def _verify_standard_wheel( + path: Path, expected_version: str = STANDARD_VERSION +) -> None: + if not path.name.endswith("-py3-none-any.whl"): + raise ValueError(f"Standard model wheel is not pure py3-none-any: {path.name}") + with zipfile.ZipFile(path) as archive: + names = _validate_zip_members(archive, path.name) + _assert_no_source_dictionaries(names, path.name) + _assert_no_python_cache(names, path.name) + metadata = _wheel_metadata(archive) + _assert_distribution_metadata( + metadata, STANDARD_NAME, expected_version, path.name + ) + if metadata.get("License-Expression") != "CC-BY-SA-3.0": + raise ValueError("Standard model data must declare CC-BY-SA-3.0") + manifest = json.loads(archive.read("radixor_models_standard/manifest.json")) + models = _validate_standard_manifest(manifest, expected_version) + dist_info = next( + name.rsplit("/", 1)[0] + for name in names + if name.endswith(".dist-info/METADATA") + ) + allowed = { + "radixor_models_standard/__init__.py", + "radixor_models_standard/manifest.json", + "radixor_models_standard/models/__init__.py", + "radixor_models_standard/notices/__init__.py", + f"{dist_info}/METADATA", + f"{dist_info}/WHEEL", + f"{dist_info}/RECORD", + f"{dist_info}/top_level.txt", + f"{dist_info}/licenses/LICENSE-MODEL-DATA.txt", + } + for model in models: + model_name = f"radixor_models_standard/{model['file']}" + notice_name = f"radixor_models_standard/{model['notice']}" + allowed.update({model_name, notice_name}) + data = archive.read(model_name) + if hashlib.sha256(data).hexdigest() != model["sha256"]: + raise ValueError(f"Checksum mismatch for {model['id']}") + if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07": + raise ValueError(f"Invalid v7 marker/version for {model['id']}") + if notice_name not in names: + raise ValueError(f"Missing model notice for {model['id']}") + unexpected = set(names) - allowed + missing = allowed - set(names) + if unexpected or missing: + raise ValueError( + f"Standard wheel allowlist mismatch; unexpected={sorted(unexpected)}, " + f"missing={sorted(missing)}" + ) + for info in archive.infolist(): + if info.filename in names and ((info.external_attr >> 16) & 0o111): + raise ValueError( + f"Standard wheel contains executable member: {info.filename}" + ) + + +def _verify_standard_sdist( + path: Path, expected_version: str = STANDARD_VERSION +) -> None: + with tarfile.open(path, "r:gz") as archive: + names = _validate_tar_members(archive, path.name) + _assert_no_source_dictionaries(names, path.name) + _assert_no_python_cache(names, path.name) + root = f"radixor_models_standard-{expected_version}" + metadata = BytesParser().parsebytes( + _tar_file_bytes(archive, f"{root}/PKG-INFO") + ) + _assert_distribution_metadata( + metadata, STANDARD_NAME, expected_version, path.name + ) + manifest_name = f"{root}/radixor_models_standard/manifest.json" + models = _validate_standard_manifest( + json.loads(_tar_file_bytes(archive, manifest_name)), expected_version + ) + egg_info = f"{root}/radixor_models_standard.egg-info" + allowed_files = { + f"{root}/LICENSE-MODEL-DATA.txt", + f"{root}/MANIFEST.in", + f"{root}/PKG-INFO", + f"{root}/README.md", + f"{root}/pyproject.toml", + f"{root}/setup.cfg", + f"{root}/radixor_models_standard/__init__.py", + manifest_name, + f"{root}/radixor_models_standard/models/__init__.py", + f"{root}/radixor_models_standard/notices/__init__.py", + f"{egg_info}/PKG-INFO", + f"{egg_info}/SOURCES.txt", + f"{egg_info}/dependency_links.txt", + f"{egg_info}/top_level.txt", + } + allowed_dirs = { + root, + f"{root}/radixor_models_standard", + f"{root}/radixor_models_standard/models", + f"{root}/radixor_models_standard/notices", + egg_info, + } + for model in models: + model_name = f"{root}/radixor_models_standard/{model['file']}" + notice_name = f"{root}/radixor_models_standard/{model['notice']}" + notice_dir = notice_name.rsplit("/", 1)[0] + allowed_files.update({model_name, notice_name}) + allowed_dirs.add(notice_dir) + data = _tar_file_bytes(archive, model_name) + if hashlib.sha256(data).hexdigest() != model["sha256"]: + raise ValueError( + f"Checksum mismatch for {model['id']} in standard sdist" + ) + if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07": + raise ValueError( + f"Invalid v7 marker/version for {model['id']} in standard sdist" + ) + files = {member.name for member in archive.getmembers() if member.isfile()} + directories = {member.name for member in archive.getmembers() if member.isdir()} + if files != allowed_files or directories != allowed_dirs: + raise ValueError( + f"Standard sdist allowlist mismatch; unexpected files={sorted(files - allowed_files)}, " + f"missing files={sorted(allowed_files - files)}, " + f"unexpected dirs={sorted(directories - allowed_dirs)}, " + f"missing dirs={sorted(allowed_dirs - directories)}" + ) + executable = [ + member.name + for member in archive.getmembers() + if member.isfile() and member.mode & 0o111 + ] + if executable: + raise ValueError( + f"Standard sdist contains executable members: {executable}" + ) + + +def _verify_fresh_install(main_wheel: Path, standard_wheel: Path) -> None: + managed_temp = REPOSITORY / "build" / "python" / "tmp" + managed_temp.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix="radixor-wheel-install-", dir=managed_temp + ) as temporary: + environment = Path(temporary) / "venv" + subprocess.run([sys.executable, "-m", "venv", str(environment)], check=True) + scripts = "Scripts" if os.name == "nt" else "bin" + python = environment / scripts / ("python.exe" if os.name == "nt" else "python") + if "radixor_models_standard-0.0.0-" in standard_wheel.name: + subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + str(standard_wheel), + str(main_wheel), + ], + check=True, + ) + else: + subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "--no-index", + "--find-links", + str(main_wheel.parent), + "--find-links", + str(standard_wheel.parent), + "radixor", + ], + check=True, + ) + result = subprocess.run( + [ + str(python), + "-c", + "from radixor import Stemmer; print(Stemmer('en').stem('running'))", + ], + check=True, + capture_output=True, + text=True, + ) + if result.stdout.strip() != "run": + raise ValueError(f"Unexpected installed model result: {result.stdout!r}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--main-wheel-dir", type=Path) + parser.add_argument("--main-sdist-dir", type=Path) + parser.add_argument("--standard-dir", type=Path) + parser.add_argument("--main-version", default=MAIN_VERSION) + parser.add_argument("--standard-version", default=STANDARD_VERSION) + parser.add_argument("--skip-install", action="store_true") + args = parser.parse_args() + + _assert_no_repository_temp_gzip() + if (args.main_wheel_dir is None) != (args.main_sdist_dir is None): + parser.error("--main-wheel-dir and --main-sdist-dir must be used together") + if args.main_wheel_dir is None and args.standard_dir is None: + parser.error("at least one distribution directory must be provided") + + main_wheel = None + standard_wheel = None + if args.main_wheel_dir is not None: + main_wheel = _one(args.main_wheel_dir, "radixor-*.whl") + main_sdist = _one(args.main_sdist_dir, "radixor-*.tar.gz") + _verify_main_wheel(main_wheel, args.main_version) + _verify_main_sdist(main_sdist, args.main_version) + if args.standard_dir is not None: + standard_wheel = _one(args.standard_dir, "radixor_models_standard-*.whl") + standard_sdist = _one(args.standard_dir, "radixor_models_standard-*.tar.gz") + _verify_standard_wheel(standard_wheel, args.standard_version) + _verify_standard_sdist(standard_sdist, args.standard_version) + if not args.skip_install and main_wheel is not None and standard_wheel is not None: + _verify_fresh_install(main_wheel, standard_wheel) + print("verified requested Radixor distributions") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/src/builder.rs b/python/src/builder.rs new file mode 100644 index 0000000..ae0ce09 --- /dev/null +++ b/python/src/builder.rs @@ -0,0 +1,682 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +// Port of the Radixor Java trie compilation pipeline +// (org.egothor.stemmer.FrequencyTrie.Builder + org.egothor.stemmer.trie.*): +// mutable trie build -> bottom-up reduction -> freeze to an immutable compiled trie. +// +// Faithful port notes: +// * Build semantics mirror StemmerPatchTrieLoader.load: for each dictionary +// entry we optionally insert the stem mapped to the NOOP patch "Na" (when +// store_original) and every variant != stem mapped to +// encode_patch(variant, stem, backward). +// * Keys are indexed per WordTraversalDirection: BACKWARD consumes characters +// right-to-left (logicalIndex = len-1-offset), FORWARD left-to-right. +// * Reduction hardcodes the production configuration verified from the Java +// source: ReductionMode = MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS, +// dominantWinnerMinPercent = 75, dominantWinnerOverSecondRatio = 3, +// contractUniformSubtrees = true (metadataForCompilation always applies +// ReductionSettings.withUniformSubtreeContraction). +// * All character/patch data is handled as UTF-16 code units (Java `char`), +// exactly as the runtime trie.rs expects. +#![allow(dead_code)] + +use std::cell::RefCell; +use std::collections::{BTreeMap, HashMap}; +use std::rc::Rc; +use std::sync::Arc; + +use crate::dict::DictEntry; +use crate::encoder::encode_patch; +use crate::patch::PatchCommand; +use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata}; + +/// Canonical no-op patch command (PatchCommandEncoder.NOOP_PATCH = "Na"). +const NOOP_PATCH: &str = "Na"; + +/// dominantWinnerMinPercent (ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT). +const DOMINANT_WINNER_MIN_PERCENT: i64 = 75; + +/// dominantWinnerOverSecondRatio (ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO). +const DOMINANT_WINNER_OVER_SECOND_RATIO: i64 = 3; + +// Ordered value-count map (Java LinkedHashMap semantics) + +/// Insertion-ordered map from a patch-command string to its accumulated local +/// frequency. Mirrors the `LinkedHashMap` used for `valueCounts` on +/// mutable nodes and `localCounts` on reduced nodes. +#[derive(Clone, Default)] +struct OrderedCounts { + entries: Vec<(String, i32)>, + index: HashMap, +} + +impl OrderedCounts { + fn new() -> Self { + OrderedCounts { + entries: Vec::new(), + index: HashMap::new(), + } + } + + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn len(&self) -> usize { + self.entries.len() + } + + /// Adds `count` to `value`, preserving first-seen insertion order. This is + /// both the build-time `put` accumulation and the reduction-time + /// `mergeLocalCounts` aggregation. + fn add(&mut self, value: &str, count: i32) { + if let Some(&position) = self.index.get(value) { + self.entries[position].1 += count; + } else { + let position = self.entries.len(); + self.index.insert(value.to_string(), position); + self.entries.push((value.to_string(), count)); + } + } +} + +// MutableNode (org.egothor.stemmer.trie.MutableNode) + +/// Mutable build-time node: children indexed by transition character plus the +/// local terminal value counts stored exactly at this node. +struct MutableNode { + children: BTreeMap, + value_counts: OrderedCounts, +} + +impl MutableNode { + fn new() -> Self { + MutableNode { + children: BTreeMap::new(), + value_counts: OrderedCounts::new(), + } + } +} + +/// Stores a value at the node addressed by `key`, incrementing its local +/// frequency by one. Mirrors `FrequencyTrie.Builder.put`. +fn put(root: &mut MutableNode, key: &[u16], value: &str, backward: bool) { + let length = key.len(); + let mut current = root; + for offset in 0..length { + // WordTraversalDirection.logicalIndex(length, offset). + let logical_index = if backward { + length - 1 - offset + } else { + offset + }; + let edge = key[logical_index]; + current = current + .children + .entry(edge) + .or_insert_with(MutableNode::new); + } + current.value_counts.add(value, 1); +} + +// ReducedNode (org.egothor.stemmer.trie.ReducedNode) + +/// Canonical reduced node used during subtree merging. Reduced nodes are shared: +/// there is exactly one instance per reduction signature, referenced through +/// `Rc` so that identical subtrees share a single instance (and therefore a +/// single frozen `Arc`). +struct ReducedNode { + /// Canonical reduction signature (see `compute_signature`). + signature: String, + /// Aggregated local value counts. + local_counts: OrderedCounts, + /// Canonical children by edge, naturally sorted ascending by the BTreeMap. + children: BTreeMap>>, + /// Whether this node is a contracted accepting leaf. + accepts: bool, +} + +impl ReducedNode { + /// Merges additional local counts into this canonical node. + fn merge_local_counts(&mut self, additional: &OrderedCounts) { + for (value, count) in &additional.entries { + self.local_counts.add(value, *count); + } + } + + /// Merges child references into this canonical node. For nodes with the same + /// reduction signature the child edge sets and child signatures are + /// compatible, so this only verifies canonical identity and stores it. + fn merge_children(&mut self, additional: &BTreeMap>>) { + for (edge, child) in additional { + match self.children.get(edge) { + Some(existing) => { + if !Rc::ptr_eq(existing, child) { + panic!("Incompatible canonical child encountered during reduction."); + } + } + None => { + self.children.insert(*edge, Rc::clone(child)); + } + } + } + } +} + +// LocalValueSummary (org.egothor.stemmer.trie.LocalValueSummary) + +/// Deterministic local terminal value summary of a node. +struct LocalValueSummary { + /// Locally stored values ordered by descending frequency, then shorter text, + /// then lexicographic (UTF-16) text, then first-seen insertion order. + ordered_values: Vec, + /// Frequencies aligned with `ordered_values` (needed for v7 serialization). + ordered_counts: Vec, + total_count: i64, + dominant_value: Option, + dominant_count: i64, + second_count: i64, +} + +impl LocalValueSummary { + /// Builds a summary from local counts, applying the exact Java ordering. + fn of(counts: &OrderedCounts) -> Self { + struct Sortable { + value: String, + count: i32, + // Java String.length() and String.compareTo operate on UTF-16 code + // units, so text ordering must compare the u16 sequence, never UTF-8. + text16: Vec, + insertion_order: usize, + } + + let mut entries: Vec = counts + .entries + .iter() + .enumerate() + .map(|(insertion_order, (value, count))| Sortable { + value: value.clone(), + count: *count, + text16: value.encode_utf16().collect(), + insertion_order, + }) + .collect(); + + entries.sort_by(|left, right| { + // 1. descending frequency + right + .count + .cmp(&left.count) + // 2. shorter text wins + .then_with(|| left.text16.len().cmp(&right.text16.len())) + // 3. lexicographically lower text (UTF-16 code units) wins + .then_with(|| left.text16.cmp(&right.text16)) + // 4. stable first-seen insertion order + .then_with(|| left.insertion_order.cmp(&right.insertion_order)) + }); + + let ordered_values: Vec = entries.iter().map(|entry| entry.value.clone()).collect(); + let ordered_counts: Vec = entries.iter().map(|entry| entry.count).collect(); + let total_count: i64 = entries.iter().map(|entry| entry.count as i64).sum(); + let dominant_value = entries.first().map(|entry| entry.value.clone()); + let dominant_count = entries.first().map(|entry| entry.count as i64).unwrap_or(0); + let second_count = entries.get(1).map(|entry| entry.count as i64).unwrap_or(0); + + LocalValueSummary { + ordered_values, + ordered_counts, + total_count, + dominant_value, + dominant_count, + second_count, + } + } + + /// Whether the dominant value satisfies both configured dominance + /// constraints (percent AND ratio), matching + /// `LocalValueSummary.hasQualifiedDominantWinner`. + fn has_qualified_dominant_winner(&self) -> bool { + if self.dominant_value.is_none() { + return false; + } + + let percent_satisfied = + self.dominant_count * 100 >= self.total_count * DOMINANT_WINNER_MIN_PERCENT; + + let ratio_satisfied = if self.second_count == 0 { + true + } else { + self.dominant_count >= self.second_count * DOMINANT_WINNER_OVER_SECOND_RATIO + }; + + percent_satisfied && ratio_satisfied + } +} + +// ReductionSignature (org.egothor.stemmer.trie.ReductionSignature and friends) + +/// Appends `text` to `buffer` using a length-prefixed, collision-free encoding +/// so arbitrary UTF-16 patch strings can be embedded without ambiguity. +fn push_len_prefixed(buffer: &mut String, text: &str) { + buffer.push_str(&text.len().to_string()); + buffer.push('#'); + buffer.push_str(text); +} + +/// Produces the canonical reduction signature of a subtree as an unambiguous +/// hashable string. Two subtrees receive equal signatures exactly when the Java +/// `ReductionSignature.equals` would consider them equal: +/// +/// * local descriptor — for DOMINANT mode this is the dominant descriptor +/// (only the dominant value) when the summary has a qualified dominant +/// winner, otherwise the ranked descriptor (the full ordered value list), +/// * whether the node accepts remaining input, +/// * the sorted list of (edge label, child signature) pairs. +fn compute_signature( + summary: &LocalValueSummary, + children: &BTreeMap>>, + accepts: bool, +) -> String { + let mut signature = String::new(); + + // Local descriptor. 'D' and 'R' markers keep a DominantLocalDescriptor + // distinct from a RankedLocalDescriptor holding the same single value, + // exactly as the Java class-based equality does. + if summary.has_qualified_dominant_winner() { + signature.push('D'); + push_len_prefixed(&mut signature, summary.dominant_value.as_ref().unwrap()); + } else { + signature.push('R'); + signature.push_str(&summary.ordered_values.len().to_string()); + signature.push(';'); + for value in &summary.ordered_values { + push_len_prefixed(&mut signature, value); + } + } + + // acceptsRemainingInput. + signature.push(if accepts { 'A' } else { 'a' }); + + // Child descriptors in sorted edge order (BTreeMap iterates ascending). + signature.push_str(&children.len().to_string()); + signature.push(';'); + for (label, child) in children { + signature.push_str(&label.to_string()); + signature.push(':'); + push_len_prefixed(&mut signature, &child.borrow().signature); + } + + signature +} + +/// Returns aggregated single-value local counts when the supplied internal +/// subtree can be contracted into an accepting leaf, otherwise `None`. +/// +/// Contraction applies (matching `FrequencyTrie.Builder.contractUniformSubtree`) +/// when the node has at least one child, every child is a single-value leaf with +/// no further children, and all those child values plus the local value (if any) +/// are the same single value. The contracted count is always 1. +fn contract_uniform_subtree( + local_counts: &OrderedCounts, + children: &BTreeMap>>, +) -> Option { + if children.is_empty() { + return None; + } + + let mut uniform_value: Option = None; + let mut value_seen = false; + + if !local_counts.is_empty() { + if local_counts.len() != 1 { + return None; + } + uniform_value = Some(local_counts.entries[0].0.clone()); + value_seen = true; + } + + for child in children.values() { + let child_ref = child.borrow(); + let is_single_value_leaf = + child_ref.children.is_empty() && child_ref.local_counts.len() == 1; + if !is_single_value_leaf { + return None; + } + let child_value = child_ref.local_counts.entries[0].0.clone(); + if value_seen && uniform_value.as_deref() != Some(child_value.as_str()) { + return None; + } + uniform_value = Some(child_value); + value_seen = true; + } + + if !value_seen { + return None; + } + + let mut contracted = OrderedCounts::new(); + contracted.add(uniform_value.as_ref().unwrap(), 1); + Some(contracted) +} + +/// Reduces a mutable node to a canonical reduced node (bottom-up). +/// +/// The order of operations mirrors the Java `reduce`: +/// 1. reduce every child first, +/// 2. try `contractUniformSubtree` (always enabled here), +/// 3. compute the local summary and reduction signature, +/// 4. deduplicate through the context map, merging counts and children into +/// an existing canonical node when the signature already exists. +fn reduce( + node: &MutableNode, + context: &mut HashMap>>, +) -> Rc> { + let mut reduced_children: BTreeMap>> = BTreeMap::new(); + for (edge, child) in node.children.iter() { + let reduced_child = reduce(child, context); + reduced_children.insert(*edge, reduced_child); + } + + let mut local_counts = node.value_counts.clone(); + let mut accepts_remaining_input = false; + + // contractUniformSubtrees is always true for the production configuration. + if let Some(contracted) = contract_uniform_subtree(&local_counts, &reduced_children) { + local_counts = contracted; + reduced_children = BTreeMap::new(); + accepts_remaining_input = true; + } + + let summary = LocalValueSummary::of(&local_counts); + let signature = compute_signature(&summary, &reduced_children, accepts_remaining_input); + + if let Some(canonical) = context.get(&signature).cloned() { + { + let mut canonical_mut = canonical.borrow_mut(); + canonical_mut.merge_local_counts(&local_counts); + canonical_mut.merge_children(&reduced_children); + } + return canonical; + } + + let canonical = Rc::new(RefCell::new(ReducedNode { + signature: signature.clone(), + local_counts, + children: reduced_children, + accepts: accepts_remaining_input, + })); + context.insert(signature, Rc::clone(&canonical)); + canonical +} + +// Freeze (FrequencyTrie.Builder.freeze -> flat CSR arrays) + +/// Maximum contiguous child-label span for which a node uses a dense +/// direct-index table instead of binary search (mirrors the Java +/// CompiledNode `maxExpandedIndex` fanout strategy). +pub(crate) const MAX_DENSE_SPAN: usize = 512; + +/// Frozen arrays of the compiled trie in CSR layout (see trie.rs). +pub(crate) struct FrozenTrie { + pub(crate) edge_start: Vec, + pub(crate) edge_labels: Vec, + pub(crate) edge_targets: Vec, + pub(crate) accepts: Vec, + pub(crate) value_start: Vec, + pub(crate) values: Vec>, + /// Patch strings parallel to `values` (needed only for serialization). + pub(crate) value_strings: Vec, + /// Frequencies parallel to `values` (needed only for v7 serialization). + pub(crate) value_counts: Vec, + pub(crate) dense_start: Vec, + pub(crate) dense_base: Vec, + pub(crate) dense_targets: Vec, +} + +/// Per-node build record collected during interning, in node-id order. +#[derive(Default)] +struct NodeBuild { + edges: Vec, + targets: Vec, + accepts: bool, + values: Vec>, + value_strings: Vec, + value_counts: Vec, +} + +/// Assigns a stable node id to each distinct canonical reduced node and records +/// its edges (ascending), child ids, and best-first values. +/// +/// Shared canonical reduced nodes (identical `Rc` allocations) are interned once +/// — the analogue of the Java `IdentityHashMap` cache +/// — so structural sharing from reduction is preserved as shared node ids. Equal +/// patch strings are compiled once and shared through `patch_cache`. +fn intern( + node: &Rc>, + index_of: &mut HashMap, + nodes: &mut Vec, + patch_cache: &mut HashMap>, + backward: bool, +) -> u32 { + let identity = Rc::as_ptr(node) as usize; + if let Some(&existing) = index_of.get(&identity) { + return existing; + } + let id = nodes.len() as u32; + index_of.insert(identity, id); + nodes.push(NodeBuild::default()); // reserve this id's slot before recursing + + let node_ref = node.borrow(); + let summary = LocalValueSummary::of(&node_ref.local_counts); + + // BTreeMap iterates ascending by edge label, so edges stay sorted. + let mut edges: Vec = Vec::with_capacity(node_ref.children.len()); + let mut targets: Vec = Vec::with_capacity(node_ref.children.len()); + for (edge, child) in node_ref.children.iter() { + edges.push(*edge); + targets.push(intern(child, index_of, nodes, patch_cache, backward)); + } + + let values: Vec> = summary + .ordered_values + .iter() + .map(|patch| { + Arc::clone( + patch_cache + .entry(patch.clone()) + .or_insert_with(|| Arc::new(PatchCommand::parse(patch, backward))), + ) + }) + .collect(); + + nodes[id as usize] = NodeBuild { + edges, + targets, + accepts: node_ref.accepts, + values, + value_strings: summary.ordered_values.clone(), + value_counts: summary.ordered_counts.clone(), + }; + id +} + +/// Freezes the reduced graph rooted at `root` (node id 0) into flat CSR arrays. +fn freeze(root: &Rc>, backward: bool) -> FrozenTrie { + let mut index_of: HashMap = HashMap::new(); + let mut nodes: Vec = Vec::new(); + let mut patch_cache: HashMap> = HashMap::new(); + intern(root, &mut index_of, &mut nodes, &mut patch_cache, backward); + + let node_count = nodes.len(); + let mut edge_start: Vec = Vec::with_capacity(node_count + 1); + let mut edge_labels: Vec = Vec::new(); + let mut edge_targets: Vec = Vec::new(); + let mut accepts: Vec = Vec::with_capacity(node_count); + let mut value_start: Vec = Vec::with_capacity(node_count + 1); + let mut values: Vec> = Vec::new(); + let mut value_strings: Vec = Vec::new(); + let mut value_counts: Vec = Vec::new(); + let mut dense_start: Vec = Vec::with_capacity(node_count + 1); + let mut dense_base: Vec = Vec::with_capacity(node_count); + let mut dense_targets: Vec = Vec::new(); + + edge_start.push(0); + value_start.push(0); + dense_start.push(0); + for nb in &nodes { + edge_labels.extend_from_slice(&nb.edges); + edge_targets.extend_from_slice(&nb.targets); + edge_start.push(edge_labels.len() as u32); + accepts.push(nb.accepts); + for v in &nb.values { + values.push(Arc::clone(v)); + } + for v in &nb.value_strings { + value_strings.push(v.clone()); + } + value_counts.extend_from_slice(&nb.value_counts); + value_start.push(values.len() as u32); + + // Decide dense vs sparse child lookup by fanout/span. + let count = nb.edges.len(); + let mut dense = false; + if count >= 2 { + let first = nb.edges[0] as usize; + let last = nb.edges[count - 1] as usize; // edges are ascending + let span = last - first + 1; + if span <= MAX_DENSE_SPAN { + let base = nb.edges[0]; + let seg = dense_targets.len(); + dense_targets.resize(seg + span, 0); + for (k, &label) in nb.edges.iter().enumerate() { + dense_targets[seg + (label - base) as usize] = nb.targets[k] + 1; + } + dense_base.push(base); + dense_start.push(dense_targets.len() as u32); + dense = true; + } + } + if !dense { + dense_base.push(0); + dense_start.push(dense_targets.len() as u32); // span 0 => sparse + } + } + + FrozenTrie { + edge_start, + edge_labels, + edge_targets, + accepts, + value_start, + values, + value_strings, + value_counts, + dense_start, + dense_base, + dense_targets, + } +} + +pub(crate) fn metadata_for(backward: bool, lowercase: bool) -> TrieMetadata { + TrieMetadata { + traversal: if backward { + TraversalDirection::Backward + } else { + TraversalDirection::Forward + }, + case_mode: if lowercase { + CaseMode::LowercaseWithLocaleRoot + } else { + CaseMode::AsIs + }, + diacritic_mode: DiacriticMode::AsIs, + } +} + +/// Build the reduced+frozen trie arrays from dictionary entries (shared by the +/// in-memory builder and the compiler). +pub(crate) fn build_frozen( + entries: &[DictEntry], + backward: bool, + store_original: bool, +) -> FrozenTrie { + let mut root = MutableNode::new(); + for entry in entries { + let stem16: Vec = entry.stem.encode_utf16().collect(); + if store_original { + put(&mut root, &stem16, NOOP_PATCH, backward); + } + for variant in &entry.variants { + if variant != &entry.stem { + let variant16: Vec = variant.encode_utf16().collect(); + let patch = encode_patch(&variant16, &stem16, backward); + put(&mut root, &variant16, &patch, backward); + } + } + } + let mut context: HashMap>> = HashMap::new(); + let reduced_root = reduce(&root, &mut context); + freeze(&reduced_root, backward) +} + +fn frozen_into_trie(frozen: FrozenTrie, metadata: TrieMetadata) -> FrequencyTrie { + FrequencyTrie::new( + frozen.edge_start, + frozen.edge_labels, + frozen.edge_targets, + frozen.accepts, + frozen.value_start, + frozen.values, + frozen.dense_start, + frozen.dense_base, + frozen.dense_targets, + metadata, + ) +} + +// Public entry point + +/// Compiles dictionary entries into a read-only patch-command trie, faithfully +/// reproducing the Java `StemmerPatchTrieLoader.load` build followed by +/// `FrequencyTrie.Builder.build` (reduce + freeze). +/// +/// * `backward` — `true` selects BACKWARD traversal (all languages except the +/// right-to-left fa/he/yi), `false` selects FORWARD. +/// * `store_original` — when `true`, each stem is inserted mapped to the NOOP +/// patch `"Na"` so the stem itself is recognised. +pub fn build_trie_from_dict( + entries: &[DictEntry], + backward: bool, + store_original: bool, + lowercase: bool, +) -> FrequencyTrie { + let frozen = build_frozen(entries, backward, store_original); + frozen_into_trie(frozen, metadata_for(backward, lowercase)) +} diff --git a/python/src/dict.rs b/python/src/dict.rs new file mode 100644 index 0000000..3c1d64a --- /dev/null +++ b/python/src/dict.rs @@ -0,0 +1,122 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +// Port of StemmerDictionaryParser (Java) — line-oriented, tab-separated dictionary. +// +// Layout: first column = canonical stem, following tab-separated columns = variants. +// Remarks: the earliest occurrence of `#` or `//` terminates the logical line. +// Case: LOWERCASE_WITH_LOCALE_ROOT lowercases the line (locale-independent here). +// Items containing any whitespace character are ignored (Java: Character.isWhitespace). + +use flate2::read::GzDecoder; +use std::io::{self, Read}; + +/// One parsed dictionary entry: a canonical stem and its accepted variants, +/// in encounter order. +pub struct DictEntry { + pub stem: String, + pub variants: Vec, +} + +/// Decompress gzipped UTF-8 dictionary bytes and parse them into entries. +/// `lowercase` mirrors CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT. +#[allow(dead_code)] // public helper; the runtime path decompresses then parse_text +pub fn parse_gz_dict(compressed: &[u8], lowercase: bool) -> io::Result> { + let mut decoder = GzDecoder::new(compressed); + let mut text = String::new(); + decoder.read_to_string(&mut text)?; + Ok(parse_text(&text, lowercase)) +} + +/// Parse an already-decompressed dictionary text. +pub fn parse_text(text: &str, lowercase: bool) -> Vec { + let mut entries = Vec::new(); + + for raw_line in text.lines() { + // stripRemark(line).trim(), then lowercase. + let stripped = strip_remark(raw_line).trim(); + if stripped.is_empty() { + continue; + } + let normalized: String = if lowercase { + stripped.to_lowercase() + } else { + stripped.to_string() + }; + if normalized.is_empty() { + continue; + } + + // split on '\t' keeping trailing empties (Java split("\t", -1)). + let mut columns = normalized.split('\t'); + + let stem = match columns.next() { + Some(c) => c.trim(), + None => continue, + }; + if stem.is_empty() || contains_whitespace(stem) { + continue; + } + + let mut variants = Vec::new(); + for col in columns { + let variant = col.trim(); + if variant.is_empty() || contains_whitespace(variant) { + continue; + } + variants.push(variant.to_string()); + } + + entries.push(DictEntry { + stem: stem.to_string(), + variants, + }); + } + + entries +} + +/// Removes a trailing remark: the earliest of `#` or `//` terminates the line. +fn strip_remark(line: &str) -> &str { + let hash = line.find('#'); + let slash = line.find("//"); + let remark = match (hash, slash) { + (None, None) => return line, + (Some(h), None) => h, + (None, Some(s)) => s, + (Some(h), Some(s)) => h.min(s), + }; + &line[..remark] +} + +/// Matches Java Character.isWhitespace closely enough for dictionary items. +#[inline] +fn contains_whitespace(item: &str) -> bool { + item.chars().any(|c| c.is_whitespace()) +} diff --git a/python/src/encoder.rs b/python/src/encoder.rs new file mode 100644 index 0000000..ee0b5d9 --- /dev/null +++ b/python/src/encoder.rs @@ -0,0 +1,349 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +// Port of PatchCommandEncoder (Java) — DP-based minimum-cost edit script. +// Costs: insert=1, delete=1, replace=1, match=0, mismatch_penalty=100. +// Produces compact opcode strings: D(elete), I(nsert), R(eplace), -(skip), N(oop). +// Count argument: 'a' + count - 1 (i.e., COUNT_SENTINEL = 'a' - 1 = 96). + +const MISMATCH_PENALTY: i32 = 100; +const COUNT_SENTINEL: u16 = b'a' as u16 - 1; // 96 = 0x60 + +#[derive(Clone, Copy, PartialEq)] +enum Trace { + Delete, + Insert, + Replace, + Match, +} + +/// Encode the patch command that transforms `source` (UTF-16 slice) into `target`. +/// Returns "Na" when source == target. +pub fn encode_patch(source: &[u16], target: &[u16], backward: bool) -> String { + if source == target { + return "Na".to_string(); + } + if backward { + encode_backward(source, target) + } else { + encode_forward(source, target) + } +} + +// Backward traversal encoding. + +fn encode_backward(source: &[u16], target: &[u16]) -> String { + let src_len = source.len(); + let tgt_len = target.len(); + let cols = tgt_len + 1; + + let mut cost = vec![0i32; (src_len + 1) * cols]; + let mut trace = vec![Trace::Match; (src_len + 1) * cols]; + + let idx = |r: usize, c: usize| r * cols + c; + + // Boundary conditions (Egothor backward: rows=source, cols=target) + for i in 1..=src_len { + cost[idx(i, 0)] = i as i32; + trace[idx(i, 0)] = Trace::Delete; + } + for j in 1..=tgt_len { + cost[idx(0, j)] = j as i32; + trace[idx(0, j)] = Trace::Insert; + } + + // Fill left-to-right, top-to-bottom (sourceIndex 1..=srcLen, targetIndex 1..=tgtLen) + for si in 1..=src_len { + let src_ch = source[si - 1]; // sourceCharacters[sourceIndex + sourceCharacterOffset=-1] + for ti in 1..=tgt_len { + let tgt_ch = target[ti - 1]; + + // sourceNeighbor = sourceIndex - 1, targetNeighbor = targetIndex - 1 + let del = cost[idx(si - 1, ti)] + 1; // DELETE from [si-1][ti] + let ins = cost[idx(si, ti - 1)] + 1; // INSERT from [si][ti-1] + let diag = cost[idx(si - 1, ti - 1)]; + let rep = diag + 1; + let mat = diag + + if src_ch == tgt_ch { + 0 + } else { + MISMATCH_PENALTY + }; + + // Priority: MATCH (baseline), then DELETE (<=), INSERT (<), REPLACE (<) + let mut best = mat; + let mut bt = Trace::Match; + if del <= best { + best = del; + bt = Trace::Delete; + } + if ins < best { + best = ins; + bt = Trace::Insert; + } + if rep < best { + bt = Trace::Replace; + } + let _ = best; + + cost[idx(si, ti)] = if bt == Trace::Replace { + rep + } else if bt == Trace::Insert { + ins + } else if bt == Trace::Delete { + del + } else { + mat + }; + trace[idx(si, ti)] = bt; + } + } + + build_patch_backward(&trace, target, cols, src_len, tgt_len) +} + +fn build_patch_backward( + trace: &[Trace], + target: &[u16], + cols: usize, + src_len: usize, + tgt_len: usize, +) -> String { + let idx = |r: usize, c: usize| r * cols + c; + + let mut patch = String::new(); + let mut pending_deletes: u16 = COUNT_SENTINEL; + let mut pending_skips: u16 = COUNT_SENTINEL; + + let mut si = src_len; + let mut ti = tgt_len; + + while si != 0 || ti != 0 { + match trace[idx(si, ti)] { + Trace::Delete => { + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + pending_deletes = pending_deletes.wrapping_add(1); + si -= 1; + } + Trace::Insert => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + ti -= 1; + append_instruction(&mut patch, 'I', target[ti]); + } + Trace::Replace => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + ti -= 1; + si -= 1; + append_instruction(&mut patch, 'R', target[ti]); + } + Trace::Match => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + pending_skips = pending_skips.wrapping_add(1); + si -= 1; + ti -= 1; + } + } + } + + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + } + + patch +} + +// Forward traversal encoding. + +fn encode_forward(source: &[u16], target: &[u16]) -> String { + let src_len = source.len(); + let tgt_len = target.len(); + let cols = tgt_len + 1; + + let mut cost = vec![0i32; (src_len + 1) * cols]; + let mut trace = vec![Trace::Match; (src_len + 1) * cols]; + + let idx = |r: usize, c: usize| r * cols + c; + + // Boundary conditions (fill from bottom-right corner) + // cost[srcLen][tgtLen] = 0, trace = MATCH + for si in (0..src_len).rev() { + cost[idx(si, tgt_len)] = cost[idx(si + 1, tgt_len)] + 1; + trace[idx(si, tgt_len)] = Trace::Delete; + } + for ti in (0..tgt_len).rev() { + cost[idx(src_len, ti)] = cost[idx(src_len, ti + 1)] + 1; + trace[idx(src_len, ti)] = Trace::Insert; + } + + // Fill right-to-left, bottom-to-top + for si in (0..src_len).rev() { + let src_ch = source[si]; // sourceCharacters[sourceIndex + sourceCharacterOffset=0] + for ti in (0..tgt_len).rev() { + let tgt_ch = target[ti]; + + // sourceNeighbor = sourceIndex + 1, targetNeighbor = targetIndex + 1 + let del = cost[idx(si + 1, ti)] + 1; + let ins = cost[idx(si, ti + 1)] + 1; + let diag = cost[idx(si + 1, ti + 1)]; + let rep = diag + 1; + let mat = diag + + if src_ch == tgt_ch { + 0 + } else { + MISMATCH_PENALTY + }; + + let mut best = mat; + let mut bt = Trace::Match; + if del <= best { + best = del; + bt = Trace::Delete; + } + if ins < best { + best = ins; + bt = Trace::Insert; + } + if rep < best { + bt = Trace::Replace; + } + let _ = best; + + cost[idx(si, ti)] = if bt == Trace::Replace { + rep + } else if bt == Trace::Insert { + ins + } else if bt == Trace::Delete { + del + } else { + mat + }; + trace[idx(si, ti)] = bt; + } + } + + build_patch_forward(&trace, target, cols, src_len, tgt_len) +} + +fn build_patch_forward( + trace: &[Trace], + target: &[u16], + cols: usize, + src_len: usize, + tgt_len: usize, +) -> String { + let idx = |r: usize, c: usize| r * cols + c; + + let mut patch = String::new(); + let mut pending_deletes: u16 = COUNT_SENTINEL; + let mut pending_skips: u16 = COUNT_SENTINEL; + + let mut si = 0usize; + let mut ti = 0usize; + + while si != src_len || ti != tgt_len { + match trace[idx(si, ti)] { + Trace::Delete => { + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + pending_deletes = pending_deletes.wrapping_add(1); + si += 1; + } + Trace::Insert => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + append_instruction(&mut patch, 'I', target[ti]); + ti += 1; + } + Trace::Replace => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + if pending_skips != COUNT_SENTINEL { + append_instruction(&mut patch, '-', pending_skips); + pending_skips = COUNT_SENTINEL; + } + append_instruction(&mut patch, 'R', target[ti]); + si += 1; + ti += 1; + } + Trace::Match => { + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + pending_deletes = COUNT_SENTINEL; + } + pending_skips = pending_skips.wrapping_add(1); + si += 1; + ti += 1; + } + } + } + + if pending_deletes != COUNT_SENTINEL { + append_instruction(&mut patch, 'D', pending_deletes); + } + + patch +} + +// Instruction encoding helpers. + +#[inline] +fn append_instruction(patch: &mut String, opcode: char, argument: u16) { + patch.push(opcode); + patch.push(char::from_u32(argument as u32).unwrap_or('\u{FFFD}')); +} diff --git a/python/src/lib.rs b/python/src/lib.rs new file mode 100644 index 0000000..5c53d80 --- /dev/null +++ b/python/src/lib.rs @@ -0,0 +1,346 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +mod builder; +mod dict; +mod encoder; +mod patch; +mod serial; +mod trie; + +use flate2::read::GzDecoder; +use pyo3::prelude::*; +use pyo3::pybacked::PyBackedStr; +use pyo3::types::{PyList, PyString}; +use std::collections::HashMap; +use std::fs; +use std::io::Read; +use std::sync::{Arc, Mutex}; +use trie::FrequencyTrie; + +/// Decompress a gzip byte image, or return the bytes unchanged when they are +/// not gzip-framed (so plain-text dictionaries also work). +fn decompress_or_raw(bytes: &[u8]) -> Vec { + if bytes.len() >= 2 && bytes[0] == 0x1F && bytes[1] == 0x8B { + let mut out = Vec::new(); + if GzDecoder::new(bytes).read_to_end(&mut out).is_ok() { + return out; + } + } + bytes.to_vec() +} + +/// Decode UTF-16 code units into a reused UTF-8 buffer (lossy on unpaired +/// surrogates, which never occur in valid patch output). +#[inline] +fn decode_utf16_into(units: &[u16], out: &mut String) { + out.clear(); + for r in char::decode_utf16(units.iter().copied()) { + out.push(r.unwrap_or('\u{FFFD}')); + } +} + +/// Runtime stemmer core: compiles a gzipped textual dictionary into a +/// patch-command trie (in Rust) and stems words against it. +#[pyclass(module = "radixor._radixor")] +struct StemmerCore { + trie: Arc, + // Optional result cache (like PyStemmer's): maps an input word to the + // already-built Python result object (a str, or None). A hit is a refcount + // bump — no re-stemming and no new string. Disabled when `cache_cap == 0`. + cache: Option>>>, + cache_cap: usize, +} + +impl StemmerCore { + fn stem_cached( + &self, + py: Python<'_>, + word: &str, + key_buf: &mut Vec, + u16_buf: &mut Vec, + u8_buf: &mut String, + ) -> Py { + let may_insert = if let Some(cache) = &self.cache { + let map = cache.lock().unwrap(); + if let Some(obj) = map.get(word) { + return obj.clone_ref(py); + } + map.len() < self.cache_cap + } else { + false + }; + + let computed: Py = match self.trie.stem_len_into(word, key_buf, u16_buf) { + Some(_) => { + decode_utf16_into(u16_buf, u8_buf); + PyString::new_bound(py, u8_buf).into_any().unbind() + } + None => py.None(), + }; + + // A full insertion-only cache cannot become writable again, so avoid + // a second lock and hash probe for later distinct words. + if may_insert { + let cache = self.cache.as_ref().expect("enabled cache"); + let mut map = cache.lock().unwrap(); + // Another thread may have populated this word while this thread + // was stemming it. Return the shared cached object when it did. + if let Some(obj) = map.get(word) { + return obj.clone_ref(py); + } + if map.len() < self.cache_cap { + map.insert(word.to_owned(), computed.clone_ref(py)); + } + } + computed + } + + fn stem_batch_impl<'py>( + &self, + py: Python<'py>, + words: &[PyBackedStr], + fallback_to_original: bool, + ) -> PyResult> { + let mut key_buf: Vec = Vec::new(); + let mut u16_buf: Vec = Vec::new(); + let mut u8_buf = String::new(); + let list = PyList::empty_bound(py); + + // Misses remain cached as None so calls through the compatibility API + // cannot change the existing stem/stem_batch missing-value contract. + for w in words { + let key: &str = w; + let obj = self.stem_cached(py, key, &mut key_buf, &mut u16_buf, &mut u8_buf); + if fallback_to_original && obj.bind(py).is_none() { + list.append(PyString::new_bound(py, key))?; + } else { + list.append(obj.bind(py))?; + } + } + Ok(list) + } +} + +#[pymethods] +impl StemmerCore { + /// Compile a model from a gzipped TSV source dictionary. + /// + /// * `path` — path to either a gzipped TSV source dictionary + /// (`stem\tvariant1\tvariant2...` per line) OR a compiled `.rxc` trie + /// (Java-interoperable v7 format). The format is auto-detected. + /// * `backward` — BACKWARD traversal (all languages except the + /// right-to-left fa/he/yi, which use FORWARD). Ignored for compiled input + /// (baked into the file). + /// * `store_original` — map each canonical stem to the no-op patch so the + /// stem itself is recognised. Ignored for compiled input. + #[new] + #[pyo3(signature = (path, backward=true, store_original=true, lowercase=true, cache_size=10_000))] + fn new( + path: &str, + backward: bool, + store_original: bool, + lowercase: bool, + cache_size: usize, + ) -> PyResult { + let raw = + fs::read(path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?; + let decompressed = decompress_or_raw(&raw); + // Auto-detect: a compiled v7 trie starts with the stream magic; anything + // else is a textual TSV dictionary compiled here in Rust. + let trie = if serial::is_v7_stream(&decompressed) { + serial::read_stream(&decompressed) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))? + } else { + // Dictionary keys are always lowercased at build time (canonical + // form). `lowercase` controls whether lookups lowercase the input at + // runtime; set it False for already-lowercased input. + let text = String::from_utf8_lossy(&decompressed); + let entries = dict::parse_text(&text, true); + builder::build_trie_from_dict(&entries, backward, store_original, lowercase) + }; + let cache = if cache_size > 0 { + // Keep PyStemmer's default entry limit without charging every + // Stemmer instance for 10,000 buckets before its first lookup. + Some(Mutex::new(HashMap::new())) + } else { + None + }; + Ok(StemmerCore { + trie: Arc::new(trie), + cache, + cache_cap: cache_size, + }) + } + + fn stem(&self, py: Python<'_>, word: &str) -> Py { + self.stem_cached( + py, + word, + &mut Vec::new(), + &mut Vec::new(), + &mut String::new(), + ) + } + + /// PyStemmer-compatible scalar API. An unrecognized word is returned + /// unchanged instead of producing None. + #[pyo3(name = "stemWord")] + fn stem_word(&self, py: Python<'_>, word: &str) -> Py { + let obj = self.stem_cached( + py, + word, + &mut Vec::new(), + &mut Vec::new(), + &mut String::new(), + ); + if obj.bind(py).is_none() { + PyString::new_bound(py, word).into_any().unbind() + } else { + obj + } + } + + fn stem_batch<'py>( + &self, + py: Python<'py>, + words: Vec, + ) -> PyResult> { + self.stem_batch_impl(py, &words, false) + } + + /// PyStemmer-compatible batch API. Unrecognized words keep their position + /// in the result and are returned unchanged. + #[pyo3(name = "stemWords")] + fn stem_words<'py>( + &self, + py: Python<'py>, + words: Vec, + ) -> PyResult> { + self.stem_batch_impl(py, &words, true) + } + + fn stem_all(&self, word: &str) -> Vec { + self.trie.stem_all(word) + } + + /// Diagnostic: full batch round-trip (marshal input, allocate one String + /// per word, build the result list) with NO stemming. Measures the + /// irreducible Python<->Rust boundary + string-allocation floor. + fn _echo_batch(&self, words: Vec) -> Vec> { + words.iter().map(|w| Some(w.to_string())).collect() + } + + /// Diagnostic: pure input marshalling (sum of byte lengths), no stemming, + /// no output strings, no result list. + fn _len_batch(&self, words: Vec) -> u64 { + words.iter().map(|w| w.len() as u64).sum() + } + + /// Diagnostic: normalize + UTF-16 encode only. + fn _encode_batch(&self, words: Vec) -> u64 { + let mut key_buf = Vec::new(); + words + .iter() + .map(|w| self.trie.bench_encode(w, &mut key_buf) as u64) + .sum() + } + + /// Diagnostic: normalize + encode + trie walk (no patch apply). + fn _encodefind_batch(&self, words: Vec) -> u64 { + let mut key_buf = Vec::new(); + let mut acc = 0u64; + for w in &words { + if self.trie.bench_find(w, &mut key_buf) { + acc += 1; + } + } + acc + } + + /// Diagnostic: full stemming algorithm (normalize + UTF-16 encode + trie + /// walk + patch apply) but returning only the summed stem length — no + /// per-word output String and no Python result list. + fn _stem_lengths_batch(&self, words: Vec) -> u64 { + let mut key_buf = Vec::new(); + let mut out_buf = Vec::new(); + let mut acc = 0u64; + for w in &words { + if let Some(n) = self.trie.stem_len_into(w, &mut key_buf, &mut out_buf) { + acc += n as u64; + } + } + acc + } + + fn stem_all_batch(&self, words: Vec) -> Vec> { + words.iter().map(|w| self.trie.stem_all(w)).collect() + } +} + +/// Compile a gzipped/plain TSV source dictionary into a Java-interoperable +/// compiled trie file (v7 format), so it can be loaded instantly later. +/// +/// * `source_path` — path to a `stemmer.gz` (or plain TSV) source dictionary. +/// * `out_path` — destination compiled file (conventionally `*.rxc`). +/// * `backward` / `store_original` / `lowercase` — same meaning as the +/// `Stemmer` constructor; baked into the compiled file. +#[pyfunction] +#[pyo3(signature = (source_path, out_path, backward=true, store_original=true, lowercase=true))] +fn compile( + source_path: &str, + out_path: &str, + backward: bool, + store_original: bool, + lowercase: bool, +) -> PyResult<()> { + let raw = + fs::read(source_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?; + let decompressed = decompress_or_raw(&raw); + if serial::is_v7_stream(&decompressed) { + return Err(pyo3::exceptions::PyValueError::new_err( + "source is already a compiled trie", + )); + } + let text = String::from_utf8_lossy(&decompressed); + let entries = dict::parse_text(&text, true); + let frozen = builder::build_frozen(&entries, backward, store_original); + let metadata = builder::metadata_for(backward, lowercase); + let bytes = serial::write_v7(&frozen, &metadata) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + fs::write(out_path, bytes).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?; + Ok(()) +} + +#[pymodule] +fn _radixor(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_function(wrap_pyfunction!(compile, m)?)?; + Ok(()) +} diff --git a/python/src/patch.rs b/python/src/patch.rs new file mode 100644 index 0000000..5f6176e --- /dev/null +++ b/python/src/patch.rs @@ -0,0 +1,494 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#[derive(Debug, Clone)] +pub enum PatchCommand { + Preserve, + DeleteSuffix(usize), + DeletePrefix(usize), + AppendChar(u16), + PrependChar(u16), + ReplaceLastChar(u16), + ReplaceFirstChar(u16), + BackwardCompound { + opcodes: Vec, + operands: Vec, + length_delta: i32, + min_len: usize, + }, + ForwardCompound { + opcodes: Vec, + operands: Vec, + length_delta: i32, + min_len: usize, + }, +} + +const SKIP: u8 = b'-'; +const DELETE: u8 = b'D'; +const INSERT: u8 = b'I'; +const REPLACE: u8 = b'R'; +const NOOP: u8 = b'N'; + +fn decode_count(arg: u16) -> Option { + if arg < b'a' as u16 { + return None; + } + Some((arg - b'a' as u16) as usize + 1) +} + +fn compile_operand(opcode: u8, arg: u16) -> Option { + match opcode { + SKIP | DELETE => { + let count = decode_count(arg)?; + if count < 1 { + None + } else { + Some(count as u32) + } + } + INSERT | REPLACE => Some(arg as u32), + NOOP => { + if arg == b'a' as u16 { + None + } else { + panic!("Invalid NOOP arg") + } + } + _ => panic!("Unknown opcode: {}", opcode as char), + } +} + +fn length_delta(opcodes: &[u8], operands: &[u32]) -> i32 { + let mut delta: i32 = 0; + for (i, &op) in opcodes.iter().enumerate() { + match op { + DELETE => delta -= operands[i] as i32, + INSERT => delta += 1, + _ => {} + } + } + delta +} + +fn backward_min_len(opcodes: &[u8], operands: &[u32]) -> usize { + let mut min_len: usize = 0; + let mut consumed_from_end: usize = 0; + for (i, &op) in opcodes.iter().enumerate() { + let operand = operands[i] as usize; + match op { + SKIP => consumed_from_end += operand, + DELETE => { + min_len = min_len.max(consumed_from_end + operand); + consumed_from_end += operand; + } + INSERT => { + min_len = min_len.max(consumed_from_end); + } + REPLACE => { + min_len = min_len.max(consumed_from_end + 1); + consumed_from_end += 1; + } + _ => {} + } + } + min_len +} + +fn forward_min_len(opcodes: &[u8], operands: &[u32]) -> usize { + let mut min_len: usize = 0; + let mut position: i32 = 0; + let mut len_delta: i32 = 0; + for (i, &op) in opcodes.iter().enumerate() { + let operand = operands[i] as i32; + match op { + SKIP => position += operand, + DELETE => { + let needed = (position + operand - len_delta).max(0) as usize; + min_len = min_len.max(needed); + len_delta -= operand; + } + INSERT => { + let needed = (position - len_delta).max(0) as usize; + min_len = min_len.max(needed); + len_delta += 1; + position += 1; + } + REPLACE => { + let needed = (position + 1 - len_delta).max(0) as usize; + min_len = min_len.max(needed); + position += 1; + } + _ => {} + } + } + min_len +} + +impl PatchCommand { + pub fn parse(patch: &str, backward: bool) -> Self { + let chars: Vec = patch.encode_utf16().collect(); + let len = chars.len(); + if len == 0 || len & 1 != 0 { + return PatchCommand::Preserve; + } + + if len == 2 { + let opcode = chars[0] as u8; + let arg = chars[1]; + return Self::compile_single(opcode, arg, backward); + } + + let op_count = len / 2; + let mut opcodes = Vec::with_capacity(op_count); + let mut operands = Vec::with_capacity(op_count); + + for i in 0..op_count { + let opcode = chars[i * 2] as u8; + let arg = chars[i * 2 + 1]; + match compile_operand(opcode, arg) { + None => return PatchCommand::Preserve, + Some(operand) => { + opcodes.push(opcode); + operands.push(operand); + } + } + } + + let ld = length_delta(&opcodes, &operands); + if backward { + let min_len = backward_min_len(&opcodes, &operands); + PatchCommand::BackwardCompound { + opcodes, + operands, + length_delta: ld, + min_len, + } + } else { + let min_len = forward_min_len(&opcodes, &operands); + PatchCommand::ForwardCompound { + opcodes, + operands, + length_delta: ld, + min_len, + } + } + } + + fn compile_single(opcode: u8, arg: u16, backward: bool) -> Self { + match opcode { + DELETE => { + let count = match decode_count(arg) { + Some(c) if c >= 1 => c, + _ => return PatchCommand::Preserve, + }; + if backward { + PatchCommand::DeleteSuffix(count) + } else { + PatchCommand::DeletePrefix(count) + } + } + INSERT => { + if backward { + PatchCommand::AppendChar(arg) + } else { + PatchCommand::PrependChar(arg) + } + } + REPLACE => { + if backward { + PatchCommand::ReplaceLastChar(arg) + } else { + PatchCommand::ReplaceFirstChar(arg) + } + } + SKIP | NOOP => PatchCommand::Preserve, + _ => panic!("Unknown opcode: {}", opcode as char), + } + } + + fn computed_length(&self, src_len: usize) -> usize { + let (ld, min_len) = match self { + PatchCommand::Preserve => (0i32, 0usize), + PatchCommand::DeleteSuffix(n) | PatchCommand::DeletePrefix(n) => (-(*n as i32), 0), + PatchCommand::AppendChar(_) | PatchCommand::PrependChar(_) => (1, 0), + PatchCommand::ReplaceLastChar(_) | PatchCommand::ReplaceFirstChar(_) => (0, 1), + PatchCommand::BackwardCompound { + length_delta, + min_len, + .. + } => (*length_delta, *min_len), + PatchCommand::ForwardCompound { + length_delta, + min_len, + .. + } => (*length_delta, *min_len), + }; + if src_len < min_len { + return src_len; + } + let applied = src_len as i32 + ld; + if applied < 1 { + src_len + } else { + applied as usize + } + } + + pub fn apply(&self, source: &[u16]) -> Vec { + let mut out = Vec::new(); + self.apply_into(source, &mut out); + out + } + + /// Apply the patch into a caller-owned buffer, avoiding a per-call + /// allocation on the hot path. `out` is cleared and overwritten. + pub fn apply_into(&self, source: &[u16], out: &mut Vec) { + let src_len = source.len(); + let out_len = self.computed_length(src_len); + out.clear(); + match self { + PatchCommand::Preserve => out.extend_from_slice(source), + PatchCommand::DeleteSuffix(_) => { + if out_len < src_len { + out.extend_from_slice(&source[..out_len]); + } else { + out.extend_from_slice(source); + } + } + PatchCommand::DeletePrefix(n) => { + if out_len < src_len { + out.extend_from_slice(&source[*n..]); + } else { + out.extend_from_slice(source); + } + } + PatchCommand::AppendChar(ch) => { + out.extend_from_slice(source); + out.push(*ch); + } + PatchCommand::PrependChar(ch) => { + out.push(*ch); + out.extend_from_slice(source); + } + PatchCommand::ReplaceLastChar(ch) => { + out.extend_from_slice(source); + if src_len != 0 { + let l = out.len(); + out[l - 1] = *ch; + } + } + PatchCommand::ReplaceFirstChar(ch) => { + out.extend_from_slice(source); + if src_len != 0 { + out[0] = *ch; + } + } + PatchCommand::BackwardCompound { + opcodes, operands, .. + } => { + if src_len < self.min_len_for_compound() || out_len < 1 { + out.extend_from_slice(source); + } else { + apply_backward_into(opcodes, operands, source, out_len, out); + } + } + PatchCommand::ForwardCompound { + opcodes, operands, .. + } => { + if src_len < self.min_len_for_compound() || out_len < 1 { + out.extend_from_slice(source); + } else { + apply_forward_into(opcodes, operands, source, out_len, out); + } + } + } + } + + fn min_len_for_compound(&self) -> usize { + match self { + PatchCommand::BackwardCompound { min_len, .. } => *min_len, + PatchCommand::ForwardCompound { min_len, .. } => *min_len, + _ => 0, + } + } +} + +fn fill_with_source(out: &mut Vec, source: &[u16]) { + out.clear(); + out.extend_from_slice(source); +} + +fn apply_backward_into( + opcodes: &[u8], + operands: &[u32], + source: &[u16], + produced_len: usize, + out: &mut Vec, +) { + let src_len = source.len(); + out.clear(); + out.resize(produced_len, 0); + let mut current_len = src_len as i32; + let mut position = src_len as i32 - 1; + let mut src_end = src_len as i32; + let mut out_end = produced_len as i32; + + for (i, &op) in opcodes.iter().enumerate() { + let operand = operands[i] as i32; + match op { + SKIP => { + let skip = operand.min(src_end); + src_end -= skip; + out_end -= skip; + if out_end < 0 { + return fill_with_source(out, source); + } + let s = src_end as usize; + let o = out_end as usize; + out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]); + position = position - operand + 1; + } + DELETE => { + let del_end_excl = position + 1; + position -= operand - 1; + if position < 0 || position > current_len || position > del_end_excl { + return fill_with_source(out, source); + } + let deleted = (del_end_excl.min(current_len) - position) as i32; + if src_end < deleted { + return fill_with_source(out, source); + } + src_end -= deleted; + current_len -= deleted; + } + INSERT => { + if position < -1 || position >= current_len || out_end <= 0 { + return fill_with_source(out, source); + } + out_end -= 1; + out[out_end as usize] = operand as u16; + current_len += 1; + position += 1; + } + REPLACE => { + if position < 0 || position >= current_len || src_end <= 0 || out_end <= 0 { + return fill_with_source(out, source); + } + src_end -= 1; + out_end -= 1; + out[out_end as usize] = operand as u16; + } + _ => return fill_with_source(out, source), + } + position -= 1; + } + + if src_end != out_end { + return fill_with_source(out, source); + } + let prefix_len = src_end as usize; + out[..prefix_len].copy_from_slice(&source[..prefix_len]); +} + +fn apply_forward_into( + opcodes: &[u8], + operands: &[u32], + source: &[u16], + produced_len: usize, + out: &mut Vec, +) { + let src_len = source.len(); + out.clear(); + out.resize(produced_len, 0); + let mut current_len = src_len as i32; + let mut position: i32 = 0; + let mut src_idx: i32 = 0; + let mut out_idx: i32 = 0; + + for (i, &op) in opcodes.iter().enumerate() { + let operand = operands[i] as i32; + match op { + SKIP => { + let skip = operand.min(src_len as i32 - src_idx); + let s = src_idx as usize; + let o = out_idx as usize; + out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]); + src_idx += skip; + out_idx += skip; + position = position + operand - 1; + } + DELETE => { + if position < 0 || position > current_len { + return fill_with_source(out, source); + } + let del_len = operand.min(current_len - position); + if src_idx + del_len > src_len as i32 { + return fill_with_source(out, source); + } + src_idx += del_len; + current_len -= del_len; + position -= 1; + } + INSERT => { + if position < 0 || position > current_len || out_idx >= produced_len as i32 { + return fill_with_source(out, source); + } + out[out_idx as usize] = operand as u16; + out_idx += 1; + current_len += 1; + } + REPLACE => { + if position < 0 + || position >= current_len + || src_idx >= src_len as i32 + || out_idx >= produced_len as i32 + { + return fill_with_source(out, source); + } + src_idx += 1; + out[out_idx as usize] = operand as u16; + out_idx += 1; + } + _ => return fill_with_source(out, source), + } + position += 1; + } + + let remaining = (src_len as i32 - src_idx) as usize; + if remaining > produced_len - out_idx as usize { + return fill_with_source(out, source); + } + let o = out_idx as usize; + let s = src_idx as usize; + out[o..o + remaining].copy_from_slice(&source[s..s + remaining]); + if out_idx as usize + remaining != produced_len { + fill_with_source(out, source); + } +} diff --git a/python/src/serial.rs b/python/src/serial.rs new file mode 100644 index 0000000..61ebba9 --- /dev/null +++ b/python/src/serial.rs @@ -0,0 +1,450 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +// Java-interoperable compiled-trie binary I/O ("v7" stream), matching +// org.egothor.stemmer.StemmerPatchTrieBinaryIO / FrequencyTrie.writeTo/readFrom. +// +// File layout = gzip( big-endian Java DataOutputStream stream ): +// i32 STREAM_MAGIC=0x45475452 ; i32 STREAM_VERSION=7 +// i32 nodeCount ; i32 rootId(=0) +// writeUTF(metadata.toTextBlock()) // Java modified UTF-8 +// i32 valueCount ; valueCount x writeUTF(patch) // value dictionary +// per node id 0..nodeCount-1: +// u8 acceptsRemainingInput +// i32 edgeCount ; edgeCount x { u16 edgeLabel ; i32 childId } +// i32 valueCount ; valueCount x { i32 valueId ; i32 count } +// +// The outer gzip framing (headers/mtime) is not byte-identical across Java and +// Rust, but the INNER stream is, and both directions gunzip+parse each other. + +use std::collections::HashMap; +use std::io::{self, Read, Write}; +use std::sync::Arc; + +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; + +use crate::builder::{FrozenTrie, MAX_DENSE_SPAN}; +use crate::patch::PatchCommand; +use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata}; + +const STREAM_MAGIC: i32 = 0x4547_5452; +const STREAM_VERSION: i32 = 7; + +// Big-endian writer helpers matching Java DataOutputStream. + +fn put_i32(out: &mut Vec, v: i32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +fn put_u16(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Java DataOutputStream.writeUTF: u16 big-endian byte length + modified UTF-8. +fn put_java_utf(out: &mut Vec, s: &str) -> io::Result<()> { + let mut bytes: Vec = Vec::with_capacity(s.len()); + for u in s.encode_utf16() { + if (0x0001..=0x007F).contains(&u) { + bytes.push(u as u8); + } else if u == 0 || (0x0080..=0x07FF).contains(&u) { + bytes.push(0xC0 | ((u >> 6) as u8 & 0x1F)); + bytes.push(0x80 | (u as u8 & 0x3F)); + } else { + bytes.push(0xE0 | ((u >> 12) as u8 & 0x0F)); + bytes.push(0x80 | ((u >> 6) as u8 & 0x3F)); + bytes.push(0x80 | (u as u8 & 0x3F)); + } + } + if bytes.len() > 0xFFFF { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "string too long for Java modified UTF-8", + )); + } + put_u16(out, bytes.len() as u16); + out.extend_from_slice(&bytes); + Ok(()) +} + +// Metadata text block, byte-identical to TrieMetadata.toTextBlock. + +fn text_block(meta: &TrieMetadata) -> String { + let forward = matches!(meta.traversal, TraversalDirection::Forward); + let case = match meta.case_mode { + CaseMode::LowercaseWithLocaleRoot => "LOWERCASE_WITH_LOCALE_ROOT", + CaseMode::AsIs => "AS_IS", + }; + let diac = match meta.diacritic_mode { + DiacriticMode::AsIs => "AS_IS", + DiacriticMode::Remove => "REMOVE", + }; + let mut s = String::with_capacity(256); + s.push_str("radixor.metadata.v1\n"); + s.push_str("formatVersion=7\n"); + s.push_str(if forward { + "traversalDirection=FORWARD\n" + } else { + "traversalDirection=BACKWARD\n" + }); + s.push_str(if forward { + "rightToLeft=true\n" + } else { + "rightToLeft=false\n" + }); + s.push_str("reductionMode=MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS\n"); + s.push_str("dominantWinnerMinPercent=75\n"); + s.push_str("dominantWinnerOverSecondRatio=3\n"); + s.push_str("contractUniformSubtrees=true\n"); + s.push_str(&format!("diacriticProcessingMode={}\n", diac)); + s.push_str(&format!("caseProcessingMode={}\n", case)); + s +} + +/// Serialize the frozen trie to the inner (uncompressed) v7 stream. +fn write_stream(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result> { + let node_count = frozen.accepts.len(); + let mut out = Vec::with_capacity(1024 + frozen.edge_labels.len() * 6); + + put_i32(&mut out, STREAM_MAGIC); + put_i32(&mut out, STREAM_VERSION); + put_i32(&mut out, node_count as i32); + put_i32(&mut out, 0); // rootId + put_java_utf(&mut out, &text_block(meta))?; + + // Value dictionary: distinct patch strings in first-occurrence order across + // nodes(id) x values(local) — frozen.value_strings is already in that order. + let mut value_id: HashMap<&str, i32> = HashMap::new(); + let mut distinct: Vec<&str> = Vec::new(); + for s in &frozen.value_strings { + if !value_id.contains_key(s.as_str()) { + value_id.insert(s.as_str(), distinct.len() as i32); + distinct.push(s.as_str()); + } + } + put_i32(&mut out, distinct.len() as i32); + for s in &distinct { + put_java_utf(&mut out, s)?; + } + + for node in 0..node_count { + out.push(if frozen.accepts[node] { 1 } else { 0 }); + + let elo = frozen.edge_start[node] as usize; + let ehi = frozen.edge_start[node + 1] as usize; + put_i32(&mut out, (ehi - elo) as i32); + for k in elo..ehi { + put_u16(&mut out, frozen.edge_labels[k]); + put_i32(&mut out, frozen.edge_targets[k] as i32); + } + + let vlo = frozen.value_start[node] as usize; + let vhi = frozen.value_start[node + 1] as usize; + put_i32(&mut out, (vhi - vlo) as i32); + for k in vlo..vhi { + let id = value_id[frozen.value_strings[k].as_str()]; + put_i32(&mut out, id); + put_i32(&mut out, frozen.value_counts[k]); + } + } + + Ok(out) +} + +/// Serialize the frozen trie to a gzip-compressed v7 file image. +pub(crate) fn write_v7(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result> { + let stream = write_stream(frozen, meta)?; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&stream)?; + encoder.finish() +} + +// Compiled-stream reader. + +struct Reader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Reader<'a> { + fn new(data: &'a [u8]) -> Self { + Reader { data, pos: 0 } + } + + fn take(&mut self, n: usize) -> io::Result<&'a [u8]> { + if self.pos + n > self.data.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected end of trie stream", + )); + } + let slice = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(slice) + } + + fn i32(&mut self) -> io::Result { + let b = self.take(4)?; + Ok(i32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + + fn u16(&mut self) -> io::Result { + let b = self.take(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + + fn u8(&mut self) -> io::Result { + Ok(self.take(1)?[0]) + } + + fn java_utf(&mut self) -> io::Result { + let len = self.u16()? as usize; + let bytes = self.take(len)?; + decode_java_utf(bytes) + } +} + +fn decode_java_utf(bytes: &[u8]) -> io::Result { + let mut units: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let b = bytes[i]; + if b & 0x80 == 0 { + units.push(b as u16); + i += 1; + } else if b & 0xE0 == 0xC0 { + if i + 1 >= bytes.len() { + return Err(malformed()); + } + let b1 = bytes[i + 1]; + units.push((((b as u16 & 0x1F) << 6) | (b1 as u16 & 0x3F)) as u16); + i += 2; + } else if b & 0xF0 == 0xE0 { + if i + 2 >= bytes.len() { + return Err(malformed()); + } + let b1 = bytes[i + 1]; + let b2 = bytes[i + 2]; + units.push(((b as u16 & 0x0F) << 12) | ((b1 as u16 & 0x3F) << 6) | (b2 as u16 & 0x3F)); + i += 3; + } else { + return Err(malformed()); + } + } + Ok(String::from_utf16_lossy(&units)) +} + +fn malformed() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "malformed modified UTF-8") +} + +fn parse_metadata(text: &str) -> TrieMetadata { + let mut traversal = TraversalDirection::Backward; + let mut case_mode = CaseMode::LowercaseWithLocaleRoot; + let mut diacritic_mode = DiacriticMode::AsIs; + for line in text.lines() { + if let Some((key, value)) = line.split_once('=') { + match key { + "traversalDirection" => { + traversal = if value == "FORWARD" { + TraversalDirection::Forward + } else { + TraversalDirection::Backward + }; + } + "caseProcessingMode" => { + case_mode = if value == "AS_IS" { + CaseMode::AsIs + } else { + CaseMode::LowercaseWithLocaleRoot + }; + } + "diacriticProcessingMode" => { + diacritic_mode = if value == "REMOVE" { + DiacriticMode::Remove + } else { + DiacriticMode::AsIs + }; + } + _ => {} + } + } + } + TrieMetadata { + traversal, + case_mode, + diacritic_mode, + } +} + +/// Rebuild dense direct-index tables from the CSR edges (same policy as freeze). +fn build_dense( + edge_start: &[u32], + edge_labels: &[u16], + edge_targets: &[u32], +) -> (Vec, Vec, Vec) { + let node_count = edge_start.len() - 1; + let mut dense_start: Vec = Vec::with_capacity(node_count + 1); + let mut dense_base: Vec = Vec::with_capacity(node_count); + let mut dense_targets: Vec = Vec::new(); + dense_start.push(0); + for node in 0..node_count { + let lo = edge_start[node] as usize; + let hi = edge_start[node + 1] as usize; + let count = hi - lo; + let mut dense = false; + if count >= 2 { + let first = edge_labels[lo] as usize; + let last = edge_labels[hi - 1] as usize; + let span = last - first + 1; + if span <= MAX_DENSE_SPAN { + let base = edge_labels[lo]; + let seg = dense_targets.len(); + dense_targets.resize(seg + span, 0); + for k in lo..hi { + dense_targets[seg + (edge_labels[k] - base) as usize] = edge_targets[k] + 1; + } + dense_base.push(base); + dense_start.push(dense_targets.len() as u32); + dense = true; + } + } + if !dense { + dense_base.push(0); + dense_start.push(dense_targets.len() as u32); + } + } + (dense_start, dense_base, dense_targets) +} + +/// Read a gzip-compressed Java v7 compiled-trie image into a runtime trie. +#[allow(dead_code)] // convenience wrapper; lib.rs decompresses then calls read_stream +pub(crate) fn read_v7(gz_bytes: &[u8]) -> io::Result { + let mut data = Vec::new(); + GzDecoder::new(gz_bytes).read_to_end(&mut data)?; + read_stream(&data) +} + +/// Whether `decompressed` (an already-gunzipped byte stream) is a v7 trie image. +pub(crate) fn is_v7_stream(decompressed: &[u8]) -> bool { + decompressed.len() >= 4 + && i32::from_be_bytes([ + decompressed[0], + decompressed[1], + decompressed[2], + decompressed[3], + ]) == STREAM_MAGIC +} + +/// Parse the inner (uncompressed) v7 stream into a runtime trie. +pub(crate) fn read_stream(data: &[u8]) -> io::Result { + let mut r = Reader::new(data); + if r.i32()? != STREAM_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bad trie stream magic", + )); + } + let version = r.i32()?; + if version != STREAM_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unsupported trie stream version {version} (expected {STREAM_VERSION})"), + )); + } + let node_count = r.i32()? as usize; + let root_id = r.i32()?; + if root_id != 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported non-zero root node id", + )); + } + let metadata = parse_metadata(&r.java_utf()?); + let backward = matches!(metadata.traversal, TraversalDirection::Backward); + + let value_table_len = r.i32()? as usize; + let mut value_table: Vec> = Vec::with_capacity(value_table_len); + for _ in 0..value_table_len { + let patch = r.java_utf()?; + value_table.push(Arc::new(PatchCommand::parse(&patch, backward))); + } + + let mut edge_start: Vec = Vec::with_capacity(node_count + 1); + let mut edge_labels: Vec = Vec::new(); + let mut edge_targets: Vec = Vec::new(); + let mut accepts: Vec = Vec::with_capacity(node_count); + let mut value_start: Vec = Vec::with_capacity(node_count + 1); + let mut values: Vec> = Vec::new(); + edge_start.push(0); + value_start.push(0); + + for _ in 0..node_count { + accepts.push(r.u8()? != 0); + let edge_count = r.i32()? as usize; + for _ in 0..edge_count { + let label = r.u16()?; + let child = r.i32()? as u32; + edge_labels.push(label); + edge_targets.push(child); + } + edge_start.push(edge_labels.len() as u32); + + let value_count = r.i32()? as usize; + for _ in 0..value_count { + let value_id = r.i32()? as usize; + let _count = r.i32()?; // frequency: not used at runtime + if value_id >= value_table.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "value id out of range", + )); + } + values.push(Arc::clone(&value_table[value_id])); + } + value_start.push(values.len() as u32); + } + + let (dense_start, dense_base, dense_targets) = + build_dense(&edge_start, &edge_labels, &edge_targets); + + Ok(FrequencyTrie::new( + edge_start, + edge_labels, + edge_targets, + accepts, + value_start, + values, + dense_start, + dense_base, + dense_targets, + metadata, + )) +} diff --git a/python/src/trie.rs b/python/src/trie.rs new file mode 100644 index 0000000..19cd686 --- /dev/null +++ b/python/src/trie.rs @@ -0,0 +1,294 @@ +// Copyright (C) 2026, Leo Galambos +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +use crate::patch::PatchCommand; +use std::borrow::Cow; +use std::sync::Arc; +use unicode_general_category::{get_general_category, GeneralCategory}; +use unicode_normalization::UnicodeNormalization; + +#[derive(Debug, Clone)] +pub enum TraversalDirection { + Backward, + Forward, +} + +#[derive(Debug, Clone)] +pub enum CaseMode { + LowercaseWithLocaleRoot, + AsIs, +} + +#[derive(Debug, Clone)] +pub enum DiacriticMode { + AsIs, + Remove, +} + +#[derive(Debug, Clone)] +pub struct TrieMetadata { + pub traversal: TraversalDirection, + pub case_mode: CaseMode, + pub diacritic_mode: DiacriticMode, +} + +/// Compiled patch-command trie in a flat, cache-friendly CSR layout. +/// +/// Instead of a graph of heap-allocated, reference-counted nodes (which forces +/// a pointer chase and a likely cache miss at every character step), the whole +/// trie is stored as a handful of contiguous arrays indexed by node id: +/// +/// * `edge_start[i] .. edge_start[i+1]` slices `edge_labels` / `edge_targets` +/// for node `i` (labels sorted ascending, so child lookup is a binary search +/// over a contiguous, cache-hot slice — no pointer chasing, no atomics), +/// * `accepts[i]` marks a contracted accepting leaf, +/// * `value_start[i] .. value_start[i+1]` slices `values` (best value first). +/// +/// Node 0 is the root. Shared (deduplicated) subtrees simply reference the same +/// node id, so structural sharing from reduction is preserved without `Arc`. +pub struct FrequencyTrie { + edge_start: Vec, + edge_labels: Vec, + edge_targets: Vec, + accepts: Vec, + value_start: Vec, + values: Vec>, + // Adaptive child lookup (mirrors the Java CompiledNode fanout strategy): + // high-fanout nodes whose child labels span a small contiguous range get a + // dense direct-index table (O(1) child access); sparse nodes fall back to + // binary search over `edge_labels`. A node `i` is dense iff + // `dense_start[i+1] > dense_start[i]`; then `dense_targets[dense_start[i] + + // (label - dense_base[i])]` holds `child_id + 1` (0 = no such edge). + dense_start: Vec, + dense_base: Vec, + dense_targets: Vec, + pub metadata: TrieMetadata, +} + +impl FrequencyTrie { + #[allow(clippy::too_many_arguments)] + pub fn new( + edge_start: Vec, + edge_labels: Vec, + edge_targets: Vec, + accepts: Vec, + value_start: Vec, + values: Vec>, + dense_start: Vec, + dense_base: Vec, + dense_targets: Vec, + metadata: TrieMetadata, + ) -> Self { + FrequencyTrie { + edge_start, + edge_labels, + edge_targets, + accepts, + value_start, + values, + dense_start, + dense_base, + dense_targets, + metadata, + } + } + + /// Normalize a lookup key (used by the rare diacritic-removal path and by + /// `stem_all`). Borrows the input when no transformation is needed. + fn normalize_key<'a>(&self, word: &'a str) -> Cow<'a, str> { + let lowered: Cow<'a, str> = + if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot) + && word.chars().any(|c| c.is_uppercase()) + { + Cow::Owned(word.to_lowercase()) + } else { + Cow::Borrowed(word) + }; + if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) { + Cow::Owned(strip_diacritics(&lowered)) + } else { + lowered + } + } + + /// Encode the normalized lookup key into `key_buf` in a single pass over the + /// input: lowercasing (when configured) is folded into the UTF-16 encoding + /// so the UTF-8 input is decoded only once and no intermediate `String` is + /// allocated. The diacritic-removal path (unused by the bundled models) + /// falls back to the general `normalize_key`. + #[inline] + fn encode_key(&self, word: &str, key_buf: &mut Vec) { + key_buf.clear(); + if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) { + let normalized = self.normalize_key(word); + key_buf.extend(normalized.encode_utf16()); + return; + } + if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot) { + let mut unit = [0u16; 2]; + for c in word.chars() { + if c.is_ascii() { + // ASCII fast path: lowercasing requires a single branch. + key_buf.push(c.to_ascii_lowercase() as u16); + } else if c.is_lowercase() { + // Already lowercase (e.g. lowercase Cyrillic/Greek): encode + // directly and skip the costly Unicode special-casing. + key_buf.extend_from_slice(c.encode_utf16(&mut unit)); + } else { + for lc in c.to_lowercase() { + key_buf.extend_from_slice(lc.encode_utf16(&mut unit)); + } + } + } + } else { + key_buf.extend(word.encode_utf16()); + } + } + + /// Find the child of `node` on `label` via binary search over the node's + /// contiguous, ascending edge-label slice. Uses unchecked indexing on + /// provably in-range offsets to drop bounds checks from the hot loop. + #[inline] + fn child(&self, node: usize, label: u16) -> Option { + // Dense high-fanout node: O(1) direct index. + // SAFETY: node and node+1 index dense_start (len = num_nodes+1). + let ds = unsafe { *self.dense_start.get_unchecked(node) } as usize; + let de = unsafe { *self.dense_start.get_unchecked(node + 1) } as usize; + if de > ds { + let base = unsafe { *self.dense_base.get_unchecked(node) }; + let idx = label.wrapping_sub(base) as usize; + if idx < de - ds { + // SAFETY: ds + idx < de <= dense_targets.len(). + let t = unsafe { *self.dense_targets.get_unchecked(ds + idx) }; + if t != 0 { + return Some((t - 1) as usize); + } + } + return None; + } + // Sparse node: binary search over the contiguous ascending edge slice. + // SAFETY: node and node+1 index edge_start (len = num_nodes+1). + let lo = unsafe { *self.edge_start.get_unchecked(node) } as usize; + let hi = unsafe { *self.edge_start.get_unchecked(node + 1) } as usize; + // SAFETY: lo <= hi <= edge_labels.len() by construction. + let labels = unsafe { self.edge_labels.get_unchecked(lo..hi) }; + match labels.binary_search(&label) { + // SAFETY: lo+pos < hi <= edge_targets.len(). + Ok(pos) => Some(unsafe { *self.edge_targets.get_unchecked(lo + pos) } as usize), + Err(_) => None, + } + } + + /// Walk the trie for `key`, returning the accepting/terminal node id. + #[inline] + fn find_node(&self, key: &[u16]) -> Option { + let mut node = 0usize; + match self.metadata.traversal { + TraversalDirection::Backward => { + for &label in key.iter().rev() { + if unsafe { *self.accepts.get_unchecked(node) } { + return Some(node); + } + node = self.child(node, label)?; + } + } + TraversalDirection::Forward => { + for &label in key.iter() { + if unsafe { *self.accepts.get_unchecked(node) } { + return Some(node); + } + node = self.child(node, label)?; + } + } + } + Some(node) + } + + #[inline] + fn preferred_value(&self, node: usize) -> Option<&Arc> { + let start = self.value_start[node] as usize; + let end = self.value_start[node + 1] as usize; + if start == end { + None + } else { + Some(&self.values[start]) + } + } + + /// Stem into caller-owned scratch buffers and return the produced length + /// without allocating an output String. This also supports diagnostics + /// that isolate the algorithm from output-String allocation. + pub fn stem_len_into( + &self, + word: &str, + key_buf: &mut Vec, + out_buf: &mut Vec, + ) -> Option { + self.encode_key(word, key_buf); + let node = self.find_node(key_buf)?; + let patch = self.preferred_value(node)?; + patch.apply_into(key_buf, out_buf); + Some(out_buf.len()) + } + + /// Diagnostic: only normalize + UTF-16 encode the key. + pub fn bench_encode(&self, word: &str, key_buf: &mut Vec) -> usize { + self.encode_key(word, key_buf); + key_buf.len() + } + + /// Diagnostic: normalize + encode + trie walk (no patch apply). + pub fn bench_find(&self, word: &str, key_buf: &mut Vec) -> bool { + self.encode_key(word, key_buf); + self.find_node(key_buf).is_some() + } + + /// Return all stems in frequency order. + pub fn stem_all(&self, word: &str) -> Vec { + let mut key_u16: Vec = Vec::new(); + self.encode_key(word, &mut key_u16); + match self.find_node(&key_u16) { + None => Vec::new(), + Some(node) => { + let start = self.value_start[node] as usize; + let end = self.value_start[node + 1] as usize; + self.values[start..end] + .iter() + .map(|p| String::from_utf16_lossy(&p.apply(&key_u16))) + .collect() + } + } + } +} + +pub fn strip_diacritics(s: &str) -> String { + s.nfd() + .filter(|ch| !matches!(get_general_category(*ch), GeneralCategory::NonspacingMark)) + .collect() +} diff --git a/python/tests/conftest.py b/python/tests/conftest.py new file mode 100644 index 0000000..c59cae4 --- /dev/null +++ b/python/tests/conftest.py @@ -0,0 +1,44 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Make the generated standard resource package available to source tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPOSITORY = Path(__file__).resolve().parents[2] +GENERATED_MODELS_DISTRIBUTION = ( + REPOSITORY / "build" / "python" / "generated" / "models-standard" +) +if GENERATED_MODELS_DISTRIBUTION.is_dir(): + sys.path.insert(0, str(GENERATED_MODELS_DISTRIBUTION)) diff --git a/python/tests/test_distribution_verifier.py b/python/tests/test_distribution_verifier.py new file mode 100644 index 0000000..486a383 --- /dev/null +++ b/python/tests/test_distribution_verifier.py @@ -0,0 +1,93 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Regression tests for Python release-archive metadata validation.""" + +from __future__ import annotations + +import sys +from email.message import Message +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from build_standard_distribution import ( # noqa: E402 + _ignore_build_artifacts, + _validate_generated_project, +) +from verify_distributions import _assert_main_dependency # noqa: E402 + + +def test_standard_model_source_tree_contains_no_generated_payload() -> None: + source = Path(__file__).resolve().parents[1] / "models-standard" + package = source / "radixor_models_standard" + + assert not (package / "manifest.json").exists() + assert not list((package / "models").glob("*.rxc")) + assert not list((package / "notices").glob("*/NOTICE-model-data.txt")) + + +def test_standard_model_build_rejects_source_skeleton() -> None: + source = Path(__file__).resolve().parents[1] / "models-standard" + + with pytest.raises(ValueError, match="manifest is missing"): + _validate_generated_project(source) + + +def test_standard_model_build_ignores_local_build_state() -> None: + names = [ + "build", + "dist", + "radixor_models_standard.egg-info", + "__pycache__", + "module.pyc", + "module.pyo", + "manifest.json", + ] + + assert _ignore_build_artifacts("unused", names) == set(names[:-1]) + + +def test_native_distribution_requires_compatible_standard_models() -> None: + metadata = Message() + metadata["Requires-Dist"] = "radixor-models-standard >=1.0, <2.0" + + _assert_main_dependency(metadata, "radixor-4.1.0.tar.gz") + + +def test_native_distribution_rejects_missing_standard_models() -> None: + metadata = Message() + + with pytest.raises(ValueError, match="standard-model dependency"): + _assert_main_dependency(metadata, "radixor-4.1.0.tar.gz") diff --git a/python/tests/test_radixor.py b/python/tests/test_radixor.py new file mode 100644 index 0000000..1657c86 --- /dev/null +++ b/python/tests/test_radixor.py @@ -0,0 +1,303 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Acceptance tests for the radixor Python extension. + +Run after building the extension: + + cd python/ + pip install maturin pytest + maturin develop --release + pytest -q + +The synthetic tests are self-contained and deterministic (no network, no +bundled data). +""" + +from __future__ import annotations + +import gzip +import inspect +from pathlib import Path + +from radixor import Stemmer + + +def _write_gz_dict(lines: list[str], tmp_path: Path) -> str: + """Write a gzipped TSV dictionary into pytest's temporary directory.""" + path = tmp_path / "dictionary.gz" + with gzip.open(path, "wt", encoding="utf-8", newline="\n") as gz: + gz.write("\n".join(lines)) + return str(path) + + +# Synthetic, deterministic pipeline tests. + + +def test_backward_suffix_stemming_roundtrip(tmp_path: Path): + # stemvariant... ; backward (suffix) stemming. + dict_lines = [ + "run\trunning\truns\tran", + "cat\tcats", + "walk\twalking\twalks\twalked", + ] + path = _write_gz_dict(dict_lines, tmp_path) + s = Stemmer(path=path, backward=True, store_original=True) + + # Every listed variant must stem back to its canonical stem. + assert s.stem("running") == "run" + assert s.stem("runs") == "run" + assert s.stem("ran") == "run" + assert s.stem("cats") == "cat" + assert s.stem("walking") == "walk" + assert s.stem("walked") == "walk" + + # store_original: the stem itself is recognised (no-op patch). + assert s.stem("run") == "run" + assert s.stem("cat") == "cat" + + +def test_store_original_controls_bare_stem_identity(tmp_path: Path): + # With a single rule and store_original=True, the stem maps to itself via + # the no-op patch, and the "cat" vs "cats" terminals carry different values + # so the trie does NOT collapse to a universal rule. + path = _write_gz_dict(["cat\tcats"], tmp_path) + s_keep = Stemmer(path=path, backward=True, store_original=True) + assert s_keep.stem("cats") == "cat" + assert s_keep.stem("cat") == "cat" + + # With store_original=False, only the single rule cats->cat is present. + # Radixor's always-on uniform-subtree contraction generalizes that lone + # rule to ALL input (this is the intended generalization behavior), so the + # bare stem is rewritten by the same delete-one-suffix rule. + s_drop = Stemmer(path=path, backward=True, store_original=False) + assert s_drop.stem("cats") == "cat" + assert s_drop.stem("cat") == "ca" # generalized: delete final char + assert s_drop.stem("dogs") == "dog" # rule applies to unseen input too + + +def test_unknown_word_returns_none(tmp_path: Path): + path = _write_gz_dict(["cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True) + assert s.stem("zzzunknown") is None + + +def test_pystemmer_scalar_api_returns_original_word_for_unknown(tmp_path: Path): + path = _write_gz_dict(["cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True) + + assert s.stemWord("cats") == "cat" + assert s.stemWord("ZzZUnknown") == "ZzZUnknown" + # The original Radixor API keeps its existing missing-value contract. + assert s.stem("ZzZUnknown") is None + + +def test_pystemmer_batch_api_returns_original_words_for_unknowns(tmp_path: Path): + path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True) + words = ["running", "Nope", "cats", "QzXqZx"] + + assert s.stemWords(words) == ["run", "Nope", "cat", "QzXqZx"] + assert s.stem_batch(words) == ["run", None, "cat", None] + + +def test_pystemmer_batch_cache_does_not_change_original_api(tmp_path: Path): + path = _write_gz_dict(["cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True, cache_size=100) + + assert s.stemWords(["Unknown", "cats", "Unknown"]) == ["Unknown", "cat", "Unknown"] + assert s.stem_batch(["Unknown", "cats", "Unknown"]) == [None, "cat", None] + + +def test_wrapper_forwards_default_cache_size_and_zero_opt_out(monkeypatch): + import radixor + + constructor_calls = [] + + class RecordingStemmerCore: + def __init__(self, *args): + constructor_calls.append(args) + + monkeypatch.setattr(radixor, "StemmerCore", RecordingStemmerCore) + + radixor.Stemmer(path="model.rxc") + radixor.Stemmer(path="model.rxc", cache_size=0) + + assert constructor_calls[0][-1] == 10_000 + assert constructor_calls[1][-1] == 0 + + +def test_native_constructor_default_cache_size(): + from radixor._radixor import StemmerCore + + assert inspect.signature(StemmerCore).parameters["cache_size"].default == 10_000 + + +def test_pystemmer_language_name_alias(): + import radixor as StemmerModule + + # Only the dependency/import line changes from PyStemmer's conventional + # ``import Stemmer; Stemmer.Stemmer("english")`` usage. + s = StemmerModule.Stemmer("english") + assert s.stemWord("running") == "run" + assert s.stemWords(["running", "unknown_word"]) == ["run", "unknown_word"] + + +def test_case_is_lowercased(tmp_path: Path): + path = _write_gz_dict(["cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True) + assert s.stem("CATS") == "cat" + assert s.stem("Cats") == "cat" + + +def test_batch_matches_scalar(tmp_path: Path): + path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True) + words = ["running", "runs", "cats", "nope", "run"] + assert s.stem_batch(words) == [s.stem(w) for w in words] + + +def test_compile_roundtrip_matches_from_text(tmp_path: Path): + import os + + import radixor + + dict_lines = [ + "run\trunning\truns\tran", + "cat\tcats", + "walk\twalking\twalks\twalked", + ] + src = _write_gz_dict(dict_lines, tmp_path) + out = src + ".rxc" + radixor.compile(src, out, backward=True) + + from_text = Stemmer(path=src, backward=True) + from_compiled = Stemmer(compiled=out) + + words = [ + "running", + "runs", + "ran", + "cats", + "walking", + "walked", + "run", + "cat", + "walk", + "unknownzzz", + ] + assert from_compiled.stem_batch(words) == from_text.stem_batch(words) + # The compiled artifact uses the gzip-wrapped EGTR v7 stream format. + import gzip + + with gzip.open(out, "rb") as fh: + assert fh.read(4) == b"EGTR" + os.unlink(out) + + +def test_cache_does_not_change_results(tmp_path: Path): + path = _write_gz_dict(["run\trunning\truns", "cat\tcats"], tmp_path) + plain = Stemmer(path=path, backward=True, cache_size=0) + cached = Stemmer(path=path, backward=True, cache_size=1000) + words = ["running", "runs", "cats", "nope", "run", "running", "cats"] + assert cached.stem_batch(words) == plain.stem_batch(words) + # Repeated lookups exercise the cache-hit path. + assert cached.stem_batch(["running"] * 5) == ["run"] * 5 + + +def test_default_cache_is_shared_across_scalar_and_batch_apis(tmp_path: Path): + root = "cacheable-root-value" + variant = "cacheable-root-values" + path = _write_gz_dict([f"{root}\t{variant}"], tmp_path) + cached = Stemmer(path=path, backward=True) + + first = cached.stem(variant) + assert first == root + assert cached.stem(variant) is first + assert cached.stemWord(variant) is first + assert cached.stem_batch([variant])[0] is first + assert cached.stemWords([variant])[0] is first + + disabled = Stemmer(path=path, backward=True, cache_size=0) + uncached_first = disabled.stem(variant) + uncached_second = disabled.stem(variant) + assert uncached_first == uncached_second == root + assert uncached_first is not uncached_second + + +def test_full_cache_keeps_existing_entries_without_admitting_new_ones(tmp_path: Path): + roots = ("first-cacheable-root", "second-cacheable-root") + variants = tuple(f"{root}-value" for root in roots) + path = _write_gz_dict( + [f"{root}\t{variant}" for root, variant in zip(roots, variants)], tmp_path + ) + stemmer = Stemmer(path=path, backward=True, cache_size=1) + + first = stemmer.stem(variants[0]) + assert stemmer.stem(variants[0]) is first + + uncached = stemmer.stem(variants[1]) + assert uncached == roots[1] + assert stemmer.stem(variants[1]) == uncached + assert stemmer.stem(variants[1]) is not uncached + assert stemmer.stem(variants[0]) is first + + +def test_lowercase_false_assumes_prelowered(tmp_path: Path): + path = _write_gz_dict(["cat\tcats"], tmp_path) + s = Stemmer(path=path, backward=True, lowercase=False) + assert s.stem("cats") == "cat" # already-lowercase input works + assert s.stem("CATS") is None # not lowercased -> no match + + +def test_forward_prefix_stemming(tmp_path: Path): + # Forward traversal handles prefix-oriented morphology (RTL languages). + path = _write_gz_dict(["kitab\talkitab\talkitabu"], tmp_path) + s = Stemmer(path=path, backward=False, store_original=True) + assert s.stem("alkitab") == "kitab" + assert s.stem("alkitabu") == "kitab" + assert s.stem("kitab") == "kitab" + + +def test_stem_all_returns_candidates(tmp_path: Path): + path = _write_gz_dict(["run\trunning", "runn\trunning"], tmp_path) + s = Stemmer(path=path, backward=True, store_original=True) + alls = s.stem_all("running") + # "running" maps to both "run" and "runn"; both must be reachable. + assert set(alls) >= {"run", "runn"} + + +# Installed standard-model smoke test. + + +def test_installed_english_compiled_model(): + s = Stemmer("en") + assert s.stem_batch(["running", "walked", "cats"]) == ["run", "walk", "cat"] diff --git a/python/tests/test_standard_models.py b/python/tests/test_standard_models.py new file mode 100644 index 0000000..370380f --- /dev/null +++ b/python/tests/test_standard_models.py @@ -0,0 +1,179 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +from __future__ import annotations + +import gzip +import hashlib +import json +from contextlib import nullcontext +from importlib import resources +from pathlib import Path + +import pytest + +import radixor + +EXPECTED_MODEL_IDS = { + "cs-cz-default", + "da-dk-default", + "de-de-default", + "es-es-default", + "fa-ir-default", + "fi-fi-default", + "fr-fr-default", + "he-il-default", + "hu-hu-default", + "it-it-default", + "nb-no-default", + "nl-nl-default", + "nn-no-default", + "pl-pl-unimorph", + "pt-pt-default", + "ru-ru-default", + "sv-se-default", + "uk-ua-default", + "us-uk-default", + "yi-default", +} + + +def _manifest() -> dict: + ref = resources.files("radixor_models_standard").joinpath("manifest.json") + return json.loads(ref.read_text(encoding="utf-8")) + + +def test_standard_manifest_model_set_versions_and_license(): + manifest = _manifest() + assert manifest["catalog_version"] == "2026.1" + assert manifest["distribution_version"] == "0.0.0" + assert manifest["format"] == {"compression": "gzip", "magic": "EGTR", "version": 7} + assert {model["id"] for model in manifest["models"]} == EXPECTED_MODEL_IDS + assert "pl-pl-polimorf" not in {model["id"] for model in manifest["models"]} + assert {model["version"] for model in manifest["models"]} == {"1.0.0"} + assert {model["provenance"]["license"] for model in manifest["models"]} == { + "CC-BY-SA-3.0" + } + + +def test_standard_artifact_checksums_and_v7_headers(): + root = resources.files("radixor_models_standard") + for model in _manifest()["models"]: + data = root.joinpath("models").joinpath(f"{model['id']}.rxc").read_bytes() + assert hashlib.sha256(data).hexdigest() == model["sha256"] + assert gzip.decompress(data)[:8] == b"EGTR\x00\x00\x00\x07" + + +def test_missing_standard_data_package_is_actionable(monkeypatch): + original_files = radixor.importlib.resources.files + + def missing(package: str): + if package == radixor._STANDARD_PACKAGE: + raise ModuleNotFoundError(package) + return original_files(package) + + monkeypatch.setattr(radixor.importlib.resources, "files", missing) + with pytest.raises( + ModuleNotFoundError, match="pip install radixor-models-standard" + ): + radixor.Stemmer("en") + + +def test_missing_standard_model_is_actionable(): + with pytest.raises(FileNotFoundError, match="not in the standard Radixor catalog"): + radixor.Stemmer("zz-zz-default") + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ( + lambda manifest: manifest.update(catalog_version="2027.1"), + "incompatible or corrupt", + ), + ( + lambda manifest: manifest.update(format={"magic": "bad"}), + "incompatible or corrupt", + ), + ( + lambda manifest: manifest.update(distribution_version="2.0.0"), + "incompatible or corrupt", + ), + ], +) +def test_incompatible_manifest_is_actionable( + tmp_path: Path, monkeypatch, mutation, message +): + manifest = _manifest() + mutation(manifest) + (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path) + with pytest.raises(RuntimeError, match=message): + radixor.Stemmer("en") + + +def test_checksum_failure_is_detected(tmp_path: Path, monkeypatch): + manifest = _manifest() + (tmp_path / "models").mkdir() + (tmp_path / "models" / "us-uk-default.rxc").write_bytes(b"not the model") + (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path) + monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext) + with pytest.raises(RuntimeError, match="SHA-256 validation"): + radixor.Stemmer("en") + + +@pytest.mark.parametrize( + ("stream_header", "message"), + [ + (b"NOPE\x00\x00\x00\x07", "EGTR format marker"), + (b"EGTR\x00\x00\x00\x08", "unsupported compiled format v8"), + ], +) +def test_format_marker_and_version_are_validated( + tmp_path: Path, monkeypatch, stream_header: bytes, message: str +): + manifest = _manifest() + model = next(item for item in manifest["models"] if item["id"] == "us-uk-default") + data = gzip.compress(stream_header, mtime=0) + model["sha256"] = hashlib.sha256(data).hexdigest() + (tmp_path / "models").mkdir() + (tmp_path / "models" / "us-uk-default.rxc").write_bytes(data) + (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path) + monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext) + with pytest.raises(RuntimeError, match=message): + radixor.Stemmer("en") + + +def test_invalid_model_id_is_rejected_before_resource_lookup(): + with pytest.raises(ValueError, match="Invalid Radixor model ID"): + radixor.Stemmer("../us-uk-default") diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java index a015eda..eb382e2 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java @@ -31,6 +31,7 @@ package org.egothor.stemmer.benchmark; import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.czechStemmer; import org.egothor.stemmer.benchmark.snowball.ext.danishStemmer; import org.egothor.stemmer.benchmark.snowball.ext.dutchStemmer; import org.egothor.stemmer.benchmark.snowball.ext.finnishStemmer; @@ -39,6 +40,8 @@ import org.egothor.stemmer.benchmark.snowball.ext.germanStemmer; import org.egothor.stemmer.benchmark.snowball.ext.hungarianStemmer; import org.egothor.stemmer.benchmark.snowball.ext.italianStemmer; import org.egothor.stemmer.benchmark.snowball.ext.norwegianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.persianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.polishStemmer; import org.egothor.stemmer.benchmark.snowball.ext.portugueseStemmer; import org.egothor.stemmer.benchmark.snowball.ext.russianStemmer; import org.egothor.stemmer.benchmark.snowball.ext.spanishStemmer; @@ -50,6 +53,11 @@ import org.egothor.stemmer.benchmark.snowball.ext.yiddishStemmer; */ enum SnowballLanguageCase { + /** + * Czech Snowball stemming over the Radixor Czech dictionary. + */ + CZECH("Czech", StemmerPatchTrieLoader.Language.CS_CZ, czechStemmer::new), + /** * Danish Snowball stemming over the Radixor Danish dictionary. */ @@ -97,6 +105,16 @@ enum SnowballLanguageCase { NORWEGIAN_NYNORSK("Norwegian Nynorsk", StemmerPatchTrieLoader.Language.NN_NO, norwegianStemmer::new, "Norwegian"), + /** + * Persian Snowball stemming over the Radixor Persian dictionary. + */ + PERSIAN("Persian", StemmerPatchTrieLoader.Language.FA_IR, persianStemmer::new), + + /** + * Polish Snowball stemming over the Radixor Polish dictionary. + */ + POLISH("Polish", StemmerPatchTrieLoader.Language.PL_PL, polishStemmer::new), + /** * Portuguese Snowball stemming over the Radixor Portuguese dictionary. */ @@ -142,6 +160,19 @@ enum SnowballLanguageCase { */ private final String luceneSnowballName; + /** + * Creates a direct-only language case not provided by the current Lucene + * Snowball implementation. + * + * @param displayLanguage human-readable language name + * @param radixorLanguage matching Radixor language resource + * @param directFactory direct Snowball stemmer factory + */ + SnowballLanguageCase(final String displayLanguage, final StemmerPatchTrieLoader.Language radixorLanguage, + final SnowballStemmerAdapter.Factory directFactory) { + this(displayLanguage, radixorLanguage, directFactory, null); + } + /** * Creates a language case. * @@ -191,6 +222,9 @@ enum SnowballLanguageCase { * @return Lucene SnowballFilter algorithm name */ String luceneSnowballName() { + if (this.luceneSnowballName == null) { + throw new IllegalStateException("Lucene Snowball does not provide " + this.displayLanguage); + } return this.luceneSnowballName; } } diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java index 4d4bf01..1cb5b90 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java @@ -116,6 +116,49 @@ public class SnowballLanguageStemmerComparisonBenchmark { } } + /** + * Shared corpus state for every official direct Snowball implementation. + * + *

+ * Czech, Persian, and Polish are available in the official Snowball 3.1.0 + * distribution but not through the Lucene SnowballFilter version used by + * this project. Keeping the direct parameter domain separate prevents JMH + * from constructing unsupported Lucene workloads. + *

+ */ + @State(Scope.Benchmark) + public static class DirectSharedState { + + /** + * Language/algorithm case under comparison. + */ + @Param({ "CZECH", "DANISH", "DUTCH", "FINNISH", "FRENCH", "GERMAN", "HUNGARIAN", "ITALIAN", + "NORWEGIAN_BOKMAL", "NORWEGIAN_NYNORSK", "PERSIAN", "POLISH", "PORTUGUESE", "RUSSIAN", + "SPANISH", "SWEDISH", "YIDDISH" }) + public String languageCaseName; + + /** + * Resolved language/algorithm case. + */ + private SnowballLanguageCase languageCase; + + /** + * Shared deterministic changed-token dictionary corpus. + */ + private String[] tokens; + + /** + * Initializes the selected direct Snowball corpus before measurement. + * + * @throws IOException if the corpus cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.languageCase = SnowballLanguageCase.valueOf(this.languageCaseName); + this.tokens = LanguageBenchmarkCorpus.createTokens(this.languageCase.radixorLanguage()); + } + } + /** * Per-thread direct Snowball state. */ @@ -130,10 +173,10 @@ public class SnowballLanguageStemmerComparisonBenchmark { /** * Initializes direct Snowball state for the selected language. * - * @param sharedState selected language state + * @param sharedState selected direct Snowball language state */ @Setup(Level.Trial) - public void setUp(final SharedState sharedState) { + public void setUp(final DirectSharedState sharedState) { this.snowballStemmer = sharedState.languageCase.createDirectStemmer(); } } @@ -198,7 +241,7 @@ public class SnowballLanguageStemmerComparisonBenchmark { /** * Runs Radixor over the selected Snowball-language corpus. * - * @param sharedState shared benchmark state + * @param sharedState shared direct Snowball benchmark state * @param blackhole result sink */ @Benchmark @@ -220,7 +263,7 @@ public class SnowballLanguageStemmerComparisonBenchmark { * @param blackhole result sink */ @Benchmark - public void snowballDirect(final SharedState sharedState, final DirectState directState, + public void snowballDirect(final DirectSharedState sharedState, final DirectState directState, final Blackhole blackhole) { final String[] tokens = sharedState.tokens; final SnowballStemmerAdapter stemmer = directState.snowballStemmer; diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java index 2bb652f..89a296e 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java @@ -179,6 +179,7 @@ public class StemmerComparisonBenchmarkQuality { "UKRAINIAN_RADIXOR", "UKRAINIAN_MORFOLOGIK_DIRECT", "UKRAINIAN_LUCENE_MORFOLOGIK_FILTER", + "SNOWBALL_CZECH_DIRECT", "SNOWBALL_DANISH_DIRECT", "SNOWBALL_DANISH_LUCENE_FILTER", "SNOWBALL_DUTCH_DIRECT", @@ -197,6 +198,8 @@ public class StemmerComparisonBenchmarkQuality { "SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER", "SNOWBALL_NORWEGIAN_NYNORSK_DIRECT", "SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER", + "SNOWBALL_PERSIAN_DIRECT", + "SNOWBALL_POLISH_DIRECT", "SNOWBALL_PORTUGUESE_DIRECT", "SNOWBALL_PORTUGUESE_LUCENE_FILTER", "SNOWBALL_RUSSIAN_DIRECT", @@ -365,6 +368,7 @@ public class StemmerComparisonBenchmarkQuality { UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA), UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA), UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA), + SNOWBALL_CZECH_DIRECT(StemmerPatchTrieLoader.Language.CS_CZ, SnowballLanguageCase.CZECH), SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), @@ -387,6 +391,8 @@ public class StemmerComparisonBenchmarkQuality { SnowballLanguageCase.NORWEGIAN_NYNORSK), SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO, SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_PERSIAN_DIRECT(StemmerPatchTrieLoader.Language.FA_IR, SnowballLanguageCase.PERSIAN), + SNOWBALL_POLISH_DIRECT(StemmerPatchTrieLoader.Language.PL_PL, SnowballLanguageCase.POLISH), SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), diff --git a/src/test/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmarkTest.java b/src/test/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmarkTest.java new file mode 100644 index 0000000..d06b3fc --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmarkTest.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Param; + +/** + * Verifies that direct-only Snowball 3.1.0 algorithms cannot enter the Lucene + * SnowballFilter benchmark domain. + */ +final class SnowballLanguageStemmerComparisonBenchmarkTest { + + /** + * Verifies the direct and Lucene parameter domains independently. + * + * @throws ReflectiveOperationException if the benchmark state contract changes + */ + @Test + void directAndLuceneParameterDomainsRemainExplicit() throws ReflectiveOperationException { + final Set directCases = parameterValues( + SnowballLanguageStemmerComparisonBenchmark.DirectSharedState.class); + final Set luceneCases = parameterValues( + SnowballLanguageStemmerComparisonBenchmark.SharedState.class); + final Set registeredCases = Arrays.stream(SnowballLanguageCase.values()) + .map(Enum::name) + .collect(Collectors.toUnmodifiableSet()); + + assertEquals(registeredCases, directCases); + assertEquals(17, directCases.size()); + assertEquals(14, luceneCases.size()); + assertEquals(Set.of("CZECH", "PERSIAN", "POLISH"), difference(directCases, luceneCases)); + + for (String luceneCase : luceneCases) { + SnowballLanguageCase.valueOf(luceneCase).luceneSnowballName(); + } + for (String directOnlyCase : difference(directCases, luceneCases)) { + assertThrows(IllegalStateException.class, + () -> SnowballLanguageCase.valueOf(directOnlyCase).luceneSnowballName()); + } + } + + /** + * Reads the declared JMH parameter values from a benchmark state. + * + * @param stateClass benchmark state class + * @return immutable parameter-value set + * @throws NoSuchFieldException if the state no longer declares the parameter + */ + private static Set parameterValues(final Class stateClass) throws NoSuchFieldException { + final Field field = stateClass.getField("languageCaseName"); + return Set.of(field.getAnnotation(Param.class).value()); + } + + /** + * Returns the values present in {@code left} but absent from {@code right}. + * + * @param left source set + * @param right excluded set + * @return immutable set difference + */ + private static Set difference(final Set left, final Set right) { + return left.stream().filter(value -> !right.contains(value)).collect(Collectors.toUnmodifiableSet()); + } +} diff --git a/src/test/java/org/egothor/stemmer/benchmark/quality/QualityStemmerMatrixTest.java b/src/test/java/org/egothor/stemmer/benchmark/quality/QualityStemmerMatrixTest.java index d486ec2..c6cdf3e 100644 --- a/src/test/java/org/egothor/stemmer/benchmark/quality/QualityStemmerMatrixTest.java +++ b/src/test/java/org/egothor/stemmer/benchmark/quality/QualityStemmerMatrixTest.java @@ -59,10 +59,13 @@ final class QualityStemmerMatrixTest { @Test @DisplayName("Candidate discovery is derived from every JMH quality candidate") void discoversEveryCandidate() { final List candidates = QualityStemmerMatrix.candidates(); - assertEquals(98, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed."); + assertEquals(102, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed."); assertTrue(candidates.stream().anyMatch(candidate -> !candidate.name().endsWith("_RADIXOR"))); assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("DA_DK_RADIXOR"))); assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("YI_RADIXOR"))); + assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_CZECH_DIRECT"))); + assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_PERSIAN_DIRECT"))); + assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_POLISH_DIRECT"))); assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_POLIMORF_RADIXOR") && candidate.resultLanguage().equals("pl-pl-polimorf"))); assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_LUCENE_STEMPEL_DIRECT") @@ -74,7 +77,7 @@ final class QualityStemmerMatrixTest { void completePublicationSelectionUsesOnlyDefaultModels() { final List candidates = StemmingQualityApplication.selectCandidates( EnumSet.allOf(Language.class), ""); - assertEquals(92, candidates.size()); + assertEquals(95, candidates.size()); assertTrue(candidates.stream().allMatch(candidate -> candidate.dictionaryModelId().equals(candidate.language().defaultModelId()))); assertTrue(candidates.stream().noneMatch(candidate -> @@ -95,7 +98,7 @@ final class QualityStemmerMatrixTest { final Path report = this.temporaryDirectory.resolve("matrix.csv"); QualityReportWriter.writeCsv(report, rows); final String text = Files.readString(report, StandardCharsets.UTF_8); - assertEquals(197, text.lines().count()); + assertEquals(205, text.lines().count()); for (Candidate candidate : QualityStemmerMatrix.candidates()) { final String prefix = "\"" + candidate.name() + "\",\"" + candidate.resultLanguage() + "\",\"\",\"\",\"\","; diff --git a/tools/parse-python-release-tag.sh b/tools/parse-python-release-tag.sh new file mode 100755 index 0000000..11e55e9 --- /dev/null +++ b/tools/parse-python-release-tag.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +tag="${1:-}" + +version_pattern='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' + +if [[ "${tag}" =~ ^python@(${version_pattern})$ ]]; then + printf 'PYTHON_DISTRIBUTION=radixor\nPYTHON_VERSION=%s\n' "${BASH_REMATCH[1]}" +elif [[ "${tag}" =~ ^python-models-standard@(${version_pattern})$ ]]; then + printf 'PYTHON_DISTRIBUTION=radixor-models-standard\nPYTHON_VERSION=%s\n' "${BASH_REMATCH[1]}" +else + echo "Invalid Python release tag: ${tag}" >&2 + exit 2 +fi diff --git a/tools/run-published-accuracy-benchmarks.sh b/tools/run-published-accuracy-benchmarks.sh new file mode 100755 index 0000000..d1ba928 --- /dev/null +++ b/tools/run-published-accuracy-benchmarks.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +report_date="${1:-$(date +%F)}" +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${project_root}" + +classpath_file="build/reports/jmh/jmh-runtime-classpath.txt" +if [[ ! -s "${classpath_file}" ]]; then + printf 'Missing %s; run ./gradlew writeJmhRuntimeClasspath --no-daemon first.\n' "${classpath_file}" >&2 + exit 1 +fi +IFS= read -r jmh_classpath < "${classpath_file}" + +tmp_dir="${project_root}/build/tmp/jmh" +report_dir="${project_root}/build/reports/jmh" +mkdir -p "${tmp_dir}" "${report_dir}" + +accuracy_include='^org\.egothor\.stemmer\.benchmark\.(EnglishHunspellStemmerComparisonBenchmarkQuality|EnglishStemmerComparisonBenchmarkQuality|HunspellStemmerComparisonBenchmarkQuality|StemmerComparisonBenchmarkQuality)\..*$' +coverage_include='^org\.egothor\.stemmer\.benchmark\.EnglishRadixorDictionaryCoverageBenchmark\.exactRootAgreement$' +accuracy_csv="${report_dir}/stemmer-accuracy-${report_date}.csv" + +common_arguments=( + -f 0 + -wi 0 + -i 1 + -r 1ms + -t 1 + -bm avgt + -tu ns + -rf csv +) + +java -Djava.io.tmpdir="${tmp_dir}" -Xms6g -Xmx6g \ + -cp "${jmh_classpath}" org.openjdk.jmh.Main \ + "${accuracy_include}" "${common_arguments[@]}" \ + -rff "${accuracy_csv}" \ + -o "${report_dir}/stemmer-accuracy-${report_date}.txt" + +for candidate in SNOWBALL_CZECH_DIRECT SNOWBALL_PERSIAN_DIRECT SNOWBALL_POLISH_DIRECT; do + if ! awk -F, -v candidate="${candidate}" ' + NR > 1 { + value = $8 + gsub(/^"|"$/, "", value) + if (value == candidate) { + found = 1 + } + } + END { exit found ? 0 : 1 } + ' "${accuracy_csv}"; then + printf 'Exact-root report omits %s.\n' "${candidate}" >&2 + exit 1 + fi + for counter in correctMatches evaluatedTokens changedCorrectMatches changedEvaluatedTokens \ + rootPreservedMatches rootEvaluatedTokens; do + if ! awk -F, -v candidate="${candidate}" -v suffix="exactRootAgreement:${counter}" ' + NR > 1 { + benchmark = $1 + value = $8 + gsub(/^"|"$/, "", benchmark) + gsub(/^"|"$/, "", value) + if (value == candidate && benchmark ~ suffix "$") { + found = 1 + } + } + END { exit found ? 0 : 1 } + ' "${accuracy_csv}"; then + printf 'Exact-root report omits %s for %s.\n' "${counter}" "${candidate}" >&2 + exit 1 + fi + done +done + +java -Djava.io.tmpdir="${tmp_dir}" -Xms6g -Xmx6g \ + -cp "${jmh_classpath}" org.openjdk.jmh.Main \ + "${coverage_include}" "${common_arguments[@]}" \ + -rff "${report_dir}/english-coverage-accuracy-${report_date}.csv" \ + -o "${report_dir}/english-coverage-accuracy-${report_date}.txt" diff --git a/tools/tests/test_update_benchmark_documentation.py b/tools/tests/test_update_benchmark_documentation.py new file mode 100644 index 0000000..8ce5d6e --- /dev/null +++ b/tools/tests/test_update_benchmark_documentation.py @@ -0,0 +1,174 @@ +############################################################################### +# Copyright (C) 2026, Leo Galambos +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +"""Tests for deterministic publication of current Java benchmark reports.""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "update-benchmark-documentation.py" +SPEC = importlib.util.spec_from_file_location("update_benchmark_documentation", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def speed_row(score: float, error: float = 1_000.0) -> dict[str, str]: + """Create the JMH fields consumed by the documentation updater.""" + return { + "Score": str(score), + "Score Error (99.9%)": str(error), + "Unit": "ns/op", + } + + +class BenchmarkDocumentationTest(unittest.TestCase): + """Covers current-report selection and measured-corpus arithmetic.""" + + def test_speed_uses_timing_token_denominator(self) -> None: + radixor = MODULE.Key("example.persianRadixor", ()) + snowball = MODULE.Key( + "example.snowballDirect", (("languageCaseName", "PERSIAN"),) + ) + data = MODULE.JmhData( + primary={ + radixor: speed_row(250_000.0), + snowball: speed_row(500_000.0), + }, + auxiliary={}, + ) + text = """## Speed + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `persianRadixor` | pending | pending | pending | pending | baseline | +| Official Snowball direct | `snowballDirect[PERSIAN]` | pending | pending | pending | pending | direct | + +## Interpretation Notes +""" + + updated = MODULE.update_speed_table(text, data, 5_000, "FA_IR") + + self.assertIn("| 0.250 | 0.001 | 50.0 | 1.000 |", updated) + self.assertIn("| 0.500 | 0.001 | 100.0 | 2.000 |", updated) + self.assertEqual( + updated, + MODULE.update_speed_table(updated, data, 5_000, "FA_IR"), + ) + + def test_language_parameter_resolves_ambiguous_method(self) -> None: + czech = MODULE.Key( + "example.luceneHunspellStemFilter", + (("languageCaseName", "CZECH"),), + ) + polish = MODULE.Key( + "example.luceneHunspellStemFilter", + (("languageCaseName", "POLISH"),), + ) + data = MODULE.JmhData( + primary={czech: speed_row(10.0), polish: speed_row(20.0)}, + auxiliary={}, + ) + + selected = MODULE.select_speed_key("luceneHunspellStemFilter", data, "CS_CZ") + + self.assertEqual(czech, selected) + + def test_partially_pending_speed_row_is_rejected(self) -> None: + text = """## Speed + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `persianRadixor` | pending | 0.001 | pending | pending | baseline | + +## Interpretation Notes +""" + + with self.assertRaisesRegex(ValueError, "Partially pending speed row"): + MODULE.update_speed_table( + text, MODULE.JmhData(primary={}, auxiliary={}), 5_000, "FA_IR" + ) + + def test_malformed_accuracy_row_is_rejected(self) -> None: + text = """## Accuracy + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Official Snowball direct | 75.00% | 62.500% | 100.000% | direct | + +## Speed +""" + + with self.assertRaisesRegex(ValueError, "Malformed accuracy row"): + MODULE.update_accuracy_table( + text, MODULE.JmhData(primary={}, auxiliary={}), "FA_IR", {} + ) + + def test_pending_accuracy_row_uses_current_auxiliary_counters(self) -> None: + key = MODULE.Key( + "example.exactRootAgreement", + (("candidateName", "SNOWBALL_PERSIAN_DIRECT"),), + ) + counters = { + "correctMatches": 75.0, + "evaluatedTokens": 100.0, + "changedCorrectMatches": 50.0, + "changedEvaluatedTokens": 80.0, + "rootPreservedMatches": 20.0, + "rootEvaluatedTokens": 20.0, + } + data = MODULE.JmhData(primary={}, auxiliary={key: counters}) + text = """## Accuracy + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Official Snowball direct | pending | pending | pending | direct | + +## Speed +""" + + updated = MODULE.update_accuracy_table(text, data, "FA_IR", {}) + + self.assertIn("| 75.000% | 62.500% | 100.000% |", updated) + self.assertEqual( + updated, + MODULE.update_accuracy_table(updated, data, "FA_IR", {}), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/update-benchmark-documentation.py b/tools/update-benchmark-documentation.py index 9c7b200..2b0ea91 100644 --- a/tools/update-benchmark-documentation.py +++ b/tools/update-benchmark-documentation.py @@ -104,7 +104,6 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument("--docs-root", type=Path, default=Path("docs")) parser.add_argument("--readme", type=Path, default=Path("README.md")) parser.add_argument("--corpus", type=Path, required=True) - parser.add_argument("--old-comparison", type=Path, required=True) parser.add_argument("--accuracy", type=Path, required=True) parser.add_argument("--speed", type=Path, required=True) parser.add_argument("--coverage-accuracy", type=Path, required=True) @@ -165,9 +164,13 @@ def read_corpora(path: Path) -> dict[str, dict[str, object]]: ) entry["commands"].append((row["Command class"], int(row["Command count"]))) if set(corpora) != set(LANGUAGES.values()): - raise ValueError(f"Corpus report languages differ from documentation languages: {sorted(corpora)}") + raise ValueError( + f"Corpus report languages differ from documentation languages: {sorted(corpora)}" + ) if any(entry["model"] == "pl-pl-polimorf" for entry in corpora.values()): - raise ValueError("The default-model corpus report must not contain pl-pl-polimorf.") + raise ValueError( + "The default-model corpus report must not contain pl-pl-polimorf." + ) return corpora @@ -180,11 +183,11 @@ def render_corpus_sections(language: str, entry: dict[str, object]) -> str: lines = [ "## Dictionary Corpus", "", - "| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens |", - "| --- | --- | --- | ---: | ---: | ---: | ---: |", + "| Model ID | Model version | Language | Dictionary rows | Complete quality tokens | Already-root tokens | Changed tokens | JMH timing tokens |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |", f"| `{entry['model']}` | `{entry['version']}` | `{language}` | {format_integer(int(entry['rows']))} | " f"{format_integer(total)} | {format_integer(int(entry['roots']))} | " - f"{format_integer(int(entry['changed']))} |", + f"{format_integer(int(entry['changed']))} | {format_integer(int(entry['timing']))} |", "", "## Radixor Patch Command Distribution", "", @@ -205,7 +208,9 @@ def render_corpus_sections(language: str, entry: dict[str, object]) -> str: f"{100.0 * count / total:.3f}% |" ) if command_total != total: - raise ValueError(f"Patch command count {command_total} differs from corpus total {total} for {language}.") + raise ValueError( + f"Patch command count {command_total} differs from corpus total {total} for {language}." + ) return "\n".join(lines) + "\n\n" @@ -252,17 +257,27 @@ def select_accuracy_key( for key, counters in data.auxiliary.items() if AUXILIARY_NAMES.issubset(counters) and language_words.issubset( - words(key.benchmark + " " + " ".join(f"{name} {value}" for name, value in key.parameters)) + words( + key.benchmark + + " " + + " ".join(f"{name} {value}" for name, value in key.parameters) + ) ) ] if not matches: - raise ValueError(f"No current JMH accuracy row matches language {language} and label {label}.") + raise ValueError( + f"No current JMH accuracy row matches language {language} and label {label}." + ) label_words = words(label) - language_words def score(key: Key) -> tuple[int, int, int, int, int]: identity_words = ( - words(key.benchmark + " " + " ".join(f"{name} {value}" for name, value in key.parameters)) + words( + key.benchmark + + " " + + " ".join(f"{name} {value}" for name, value in key.parameters) + ) - language_words ) return ( @@ -273,9 +288,13 @@ def select_accuracy_key( int(language_words.issubset(words(key.benchmark))), ) - ranked = sorted(((score(key), key) for key in matches), reverse=True, key=lambda item: item[0]) + ranked = sorted( + ((score(key), key) for key in matches), reverse=True, key=lambda item: item[0] + ) if ranked[0][0][0] == 0: - raise ValueError(f"No implementation identity words match accuracy label {label} for {language}.") + raise ValueError( + f"No implementation identity words match accuracy label {label} for {language}." + ) if len(ranked) > 1 and ranked[0][0] == ranked[1][0]: raise ValueError( f"Ambiguous current JMH accuracy identity for {label} in {language}: " @@ -300,11 +319,31 @@ def update_accuracy_table( ) -> str: start = text.index("## Accuracy") end = text.index("## Speed", start) - section = text[start:end] + section = text[start:end].rstrip() output: list[str] = [] for line in section.splitlines(): cells = [cell.strip() for cell in line.split("|")[1:-1]] - if len(cells) == 5 and all(re.fullmatch(r"\d+\.\d{3}%", cell) for cell in cells[1:4]): + measured = len(cells) == 5 and all( + re.fullmatch(r"\d+\.\d{3}%", cell) for cell in cells[1:4] + ) + pending = len(cells) == 5 and all(cell == "pending" for cell in cells[1:4]) + partial_pending = ( + len(cells) == 5 + and any(cell == "pending" for cell in cells[1:4]) + and not pending + ) + if partial_pending: + raise ValueError( + f"Partially pending accuracy row for {cells[0]} in {language}." + ) + if ( + len(cells) == 5 + and cells[0] not in {"Stemmer", "---"} + and not measured + and not pending + ): + raise ValueError(f"Malformed accuracy row for {cells[0]} in {language}.") + if measured or pending: if cells[0] == "Radixor": values = rounded_accuracy(corpus_accuracy(corpus)) else: @@ -312,12 +351,8 @@ def update_accuracy_table( values = rounded_accuracy(accuracy(new_data, key)) cells[1:4] = [f"{value}%" for value in values] line = "| " + " | ".join(cells) + " |" - elif language == "HE_IL" and len(cells) == 5 and cells[0] == "Radixor" and cells[1] == "pending": - values = rounded_accuracy(corpus_accuracy(corpus)) - cells[1:4] = [f"{value}%" for value in values] - line = "| " + " | ".join(cells) + " |" output.append(line) - replacement = "\n".join(output) + "\n\n" + replacement = "\n".join(output).rstrip() + "\n\n" return text[:start] + replacement + text[end:] @@ -328,9 +363,9 @@ def method_and_parameter(display: str) -> tuple[str, str]: return match.group(1), match.group(2) or "" -def speed_matches(display: str, data: JmhData) -> list[Key]: +def speed_matches(display: str, data: JmhData, language: str) -> list[Key]: method, language_case = method_and_parameter(display) - return [ + matches = [ key for key, row in data.primary.items() if key.method == method @@ -338,57 +373,54 @@ def speed_matches(display: str, data: JmhData) -> list[Key]: and key not in data.auxiliary and row["Unit"] == "ns/op" ] + if language_case or len(matches) <= 1: + return matches + language_words = LANGUAGE_IDENTITY_WORDS[language] + return [ + key + for key in matches + if language_words.issubset( + words(" ".join(value for _, value in key.parameters)) + ) + ] -def closest_speed_key(display: str, score_ms: float, data: JmhData) -> tuple[Key, float]: - matches = speed_matches(display, data) - if not matches: - raise ValueError(f"No JMH speed row matches {display}") - selected = min(matches, key=lambda key: abs(float(data.primary[key]["Score"]) / 1_000_000.0 - score_ms)) - difference = abs(float(data.primary[selected]["Score"]) / 1_000_000.0 - score_ms) - return selected, difference - - -def select_speed_key(display: str, published_score_ms: float, old_data: JmhData, new_data: JmhData) -> Key: - current, current_difference = closest_speed_key(display, published_score_ms, new_data) - if current_difference < 0.001: - return current - selected, difference = closest_speed_key(display, published_score_ms, old_data) - if difference >= 0.001: - raise ValueError(f"Old speed row for {display} differs by {difference:.6f} ms from documentation.") - return selected +def select_speed_key(display: str, data: JmhData, language: str) -> Key: + matches = speed_matches(display, data, language) + if len(matches) != 1: + raise ValueError( + f"Expected one current JMH speed row for {display} in {language}, found {len(matches)}." + ) + return matches[0] def update_speed_table( text: str, - old_data: JmhData, new_data: JmhData, - changed_tokens: int, + timing_tokens: int, language: str, ) -> str: start = text.index("## Speed") end = text.index("## Interpretation Notes", start) - section = text[start:end] + section = text[start:end].rstrip() parsed: list[tuple[str, list[str] | None, Key | None]] = [] radixor_score = math.nan for line in section.splitlines(): cells = [cell.strip() for cell in line.split("|")[1:-1]] if len(cells) == 7 and cells[1].startswith("`") and cells[1].endswith("`"): display = cells[1].strip("`") - if cells[2] == "pending" and language == "HE_IL": - matches = [ - key - for key, row in new_data.primary.items() - if key.method == "hebrewRadixor" and key not in new_data.auxiliary and row["Unit"] == "ns/op" - ] - if len(matches) != 1: - raise ValueError(f"Expected one Hebrew speed row, found {len(matches)}") - key = matches[0] - elif re.fullmatch(r"\d+\.\d{3}", cells[2]): - key = select_speed_key(display, float(cells[2]), old_data, new_data) - else: - parsed.append((line, None, None)) - continue + pending = all(cell == "pending" for cell in cells[2:6]) + measured = all(re.fullmatch(r"\d+\.\d+", cell) for cell in cells[2:6]) + partial_pending = ( + any(cell == "pending" for cell in cells[2:6]) and not pending + ) + if partial_pending: + raise ValueError( + f"Partially pending speed row for {cells[0]} in {language}." + ) + if not pending and not measured: + raise ValueError(f"Malformed speed row for {cells[0]} in {language}.") + key = select_speed_key(display, new_data, language) if key not in new_data.primary: raise ValueError(f"New JMH report omits speed key {key}") score = float(new_data.primary[key]["Score"]) @@ -408,18 +440,17 @@ def update_speed_table( error = float(row["Score Error (99.9%)"]) cells[2] = f"{score / 1_000_000.0:.3f}" cells[3] = f"{error / 1_000_000.0:.3f}" - cells[4] = f"{score / changed_tokens:.1f}" + cells[4] = f"{score / timing_tokens:.1f}" cells[5] = f"{score / radixor_score:.3f}" line = "| " + " | ".join(cells) + " |" output.append(line) - replacement = "\n".join(output) + "\n\n" + replacement = "\n".join(output).rstrip() + "\n\n" return text[:start] + replacement + text[end:] def update_language_pages( docs_root: Path, corpora: dict[str, dict[str, object]], - old_data: JmhData, accuracy_data: JmhData, speed_data: JmhData, ) -> None: @@ -429,7 +460,11 @@ def update_language_pages( text = path.read_text(encoding="utf-8") corpus_start = text.index("## Dictionary Corpus") accuracy_start = text.index("## Accuracy", corpus_start) - text = text[:corpus_start] + render_corpus_sections(language, corpora[language]) + text[accuracy_start:] + text = ( + text[:corpus_start] + + render_corpus_sections(language, corpora[language]) + + text[accuracy_start:] + ) text = re.sub( r"Speed uses JMH average time, \d+ warmup iterations, \d+ measurement iterations, " r"\d+ forks?, and 1 thread\.", @@ -439,11 +474,15 @@ def update_language_pages( count=1, ) text = update_accuracy_table(text, accuracy_data, language, corpora[language]) - text = update_speed_table(text, old_data, speed_data, int(corpora[language]["changed"]), language) + text = update_speed_table( + text, speed_data, int(corpora[language]["timing"]), language + ) path.write_text(text, encoding="utf-8") -def update_corpora_reference(docs_root: Path, corpora: dict[str, dict[str, object]]) -> None: +def update_corpora_reference( + docs_root: Path, corpora: dict[str, dict[str, object]] +) -> None: path = docs_root / "benchmarks" / "reference" / "corpora.md" text = path.read_text(encoding="utf-8") original_header = "| Language resource |" @@ -453,7 +492,9 @@ def update_corpora_reference(docs_root: Path, corpora: dict[str, dict[str, objec elif current_header in text: table_start = text.index(current_header) else: - raise ValueError("The corpora reference contains no recognized corpus-table header.") + raise ValueError( + "The corpora reference contains no recognized corpus-table header." + ) table_end = text.index("\n\n", table_start) lines = [ "| Default model ID | Version | SHA-256 | Language | Dictionary rows | Total tokens | Already-root tokens | Changed tokens | Speed timing tokens |", @@ -468,7 +509,9 @@ def update_corpora_reference(docs_root: Path, corpora: dict[str, dict[str, objec f"{format_integer(int(entry['changed']))} | {format_integer(int(entry['timing']))} |" ) replacement = "\n".join(lines) - path.write_text(text[:table_start] + replacement + text[table_end:], encoding="utf-8") + path.write_text( + text[:table_start] + replacement + text[table_end:], encoding="utf-8" + ) def coverage_rows(accuracy_data: JmhData, speed_data: JmhData) -> list[str]: @@ -558,23 +601,31 @@ def update_coverage( "quality/speed envelope: the amount and quality of dictionary knowledge affect stemming precision,\n" "while contracted tries reduce lookup cost in uniform regions of the compiled graph.\n\n" ) - index.write_text(index_text[:key_start] + key_section + index_text[key_end:], encoding="utf-8") + index.write_text( + index_text[:key_start] + key_section + index_text[key_end:], encoding="utf-8" + ) def main() -> None: arguments = parse_arguments() corpora = read_corpora(arguments.corpus) - old_data = read_jmh(arguments.old_comparison) accuracy_data = read_jmh(arguments.accuracy) speed_data = read_jmh(arguments.speed) coverage_accuracy_data = read_jmh(arguments.coverage_accuracy) coverage_speed_data = read_jmh(arguments.coverage_speed) measured_keys = set(accuracy_data.primary) | set(speed_data.primary) if any("PolishPolimorf" in key.benchmark for key in measured_keys): - raise ValueError("A published report contains the excluded PolishPolimorf benchmark.") - update_language_pages(arguments.docs_root, corpora, old_data, accuracy_data, speed_data) + raise ValueError( + "A published report contains the excluded PolishPolimorf benchmark." + ) + update_language_pages(arguments.docs_root, corpora, accuracy_data, speed_data) update_corpora_reference(arguments.docs_root, corpora) - update_coverage(arguments.docs_root, arguments.readme, coverage_accuracy_data, coverage_speed_data) + update_coverage( + arguments.docs_root, + arguments.readme, + coverage_accuracy_data, + coverage_speed_data, + ) if __name__ == "__main__":