From 38620d7e71a55579ee5241bd4998b7a6dfe2a822 Mon Sep 17 00:00:00 2001 From: Leo Galambos Date: Fri, 3 Jul 2026 18:44:39 +0200 Subject: [PATCH] feat: prepare Radixor 3.0.0 with contracted tries and compiled patch commands Introduce contracted compiled patch tries for faster lookup, make compiled patch commands the primary runtime path, refresh stemmer benchmarks and documentation, and restructure the documentation for 3.0.0 onboarding. BREAKING CHANGE: Radixor 3.0.0 promotes compiled patch-command APIs and new compiled trie artifacts as the primary runtime integration model. --- .github/workflows/benchmarks.yml | 4 +- .github/workflows/pages.yml | 13 +- README.md | 53 +- build.gradle | 14 +- docs/architecture.md | 13 +- docs/benchmarking.md | 170 +-- docs/benchmarks/index.md | 55 + docs/benchmarks/languages/czech.md | 51 + docs/benchmarks/languages/danish.md | 53 + docs/benchmarks/languages/dutch.md | 53 + docs/benchmarks/languages/english.md | 67 + docs/benchmarks/languages/finnish.md | 55 + docs/benchmarks/languages/french.md | 57 + docs/benchmarks/languages/german.md | 59 + docs/benchmarks/languages/hungarian.md | 55 + docs/benchmarks/languages/index.md | 45 + docs/benchmarks/languages/italian.md | 54 + docs/benchmarks/languages/norwegian-bokmal.md | 57 + .../benchmarks/languages/norwegian-nynorsk.md | 53 + docs/benchmarks/languages/persian.md | 49 + docs/benchmarks/languages/polish.md | 55 + docs/benchmarks/languages/portuguese.md | 59 + docs/benchmarks/languages/russian.md | 55 + docs/benchmarks/languages/spanish.md | 59 + docs/benchmarks/languages/swedish.md | 57 + docs/benchmarks/languages/ukrainian.md | 53 + docs/benchmarks/languages/yiddish.md | 52 + docs/benchmarks/reference/candidates.md | 23 + docs/benchmarks/reference/corpora.md | 28 + docs/benchmarks/reference/english-coverage.md | 43 + docs/benchmarks/reference/environment.md | 54 + docs/benchmarks/reference/methodology.md | 59 + docs/built-in-languages.md | 79 +- docs/cli-compilation.md | 14 +- docs/compatibility-and-guarantees.md | 11 +- docs/fast-track.md | 130 ++ docs/index.md | 8 +- docs/integration-deep-dive.md | 235 +++ docs/lookup-edge-optimization.md | 15 +- docs/migration-and-backward-compatibility.md | 155 ++ .../programmatic-extending-and-persistence.md | 3 +- docs/programmatic-loading-and-building.md | 30 +- docs/programmatic-querying-and-ambiguity.md | 29 +- docs/programmatic-usage.md | 13 +- docs/quick-start.md | 44 +- docs/reduction-semantics.md | 18 + docs/reports.md | 5 +- docs/test-taxonomy-and-filtering.md | 10 +- gradle.lockfile | 10 + gradle/lucene-benchmarks.gradle | 223 +++ gradle/opennlp-benchmarks.gradle | 5 + gradle/paicehusk-benchmarks.gradle | 219 +++ gradle/snowball-benchmarks.gradle | 55 +- gradle/verification-metadata.xml | 172 +-- mkdocs.yml | 49 +- .../benchmark/BenchmarkCorpusSupport.java | 4 +- .../benchmark/BenchmarkTokenSequence.java | 185 +++ .../benchmark/BenchmarkTokenStream.java | 146 ++ .../benchmark/EnglishComparisonCorpus.java | 98 +- ...ishRadixorDictionaryCoverageBenchmark.java | 431 ++++++ .../EnglishStemmerComparisonBenchmark.java | 466 +++++- ...lishStemmerComparisonBenchmarkQuality.java | 258 ++++ .../EnglishStemmerComparisonTokenStream.java | 145 ++ .../FrequencyTrieLookupBenchmark.java | 1 + .../benchmark/LanguageBenchmarkCorpus.java | 445 ++++++ ...ltiLanguageStemmerComparisonBenchmark.java | 1114 ++++++++++++++ .../benchmark/RadixorBenchmarkStemmer.java | 82 ++ .../benchmark/SnowballLanguageCase.java | 196 +++ ...allLanguageStemmerComparisonBenchmark.java | 246 ++++ .../benchmark/SnowballStemmerAdapter.java | 2 +- .../StemmerComparisonBenchmarkQuality.java | 813 +++++++++++ .../java/org/egothor/stemmer/Compile.java | 1 + .../egothor/stemmer/CompiledPatchCommand.java | 1282 +++++++++++++++++ .../org/egothor/stemmer/FrequencyTrie.java | 240 ++- .../stemmer/FrequencyTrieBuilders.java | 99 ++ .../egothor/stemmer/PatchCommandEncoder.java | 202 +-- .../egothor/stemmer/ReductionSettings.java | 41 +- .../stemmer/StemmerKnowledgeExperiment.java | 1 + .../stemmer/StemmerPatchTrieLoader.java | 506 ++++++- .../org/egothor/stemmer/TrieMetadata.java | 7 +- .../egothor/stemmer/trie/CompiledNode.java | 43 +- .../org/egothor/stemmer/trie/ReducedNode.java | 35 +- .../stemmer/trie/ReductionSignature.java | 37 +- .../stemmer/CompileIntegrationTest.java | 1 + .../stemmer/CompiledPatchCommandTest.java | 341 +++++ .../CompiledTrieArtifactRegressionTest.java | 1 + .../stemmer/FrequencyTrieBuildersTest.java | 33 +- .../egothor/stemmer/FrequencyTrieTest.java | 118 ++ .../FuzzStemmerAndTrieCompilationTest.java | 1 + .../PatchCommandEncoderProperties.java | 1 + .../stemmer/PatchCommandEncoderTest.java | 1 + .../stemmer/RegressionArtifactGenerator.java | 1 + .../stemmer/RegressionArtifactSupport.java | 1 + .../stemmer/StemmerPatchTrieLoaderTest.java | 157 ++ .../stemmer/StemmerPatchTrieProperties.java | 1 + .../benchmark/BenchmarkTokenSequenceTest.java | 187 +++ ...glishStemmerComparisonTokenStreamTest.java | 230 +++ .../LanguageBenchmarkCorpusTest.java | 228 +++ .../LucenePorterStemmerCopiedTest.java | 127 ++ .../PaiceHuskLancasterStemmerTest.java | 148 ++ tools/generate-pages-badges.py | 100 +- 101 files changed, 11235 insertions(+), 727 deletions(-) create mode 100644 docs/benchmarks/index.md create mode 100644 docs/benchmarks/languages/czech.md create mode 100644 docs/benchmarks/languages/danish.md create mode 100644 docs/benchmarks/languages/dutch.md create mode 100644 docs/benchmarks/languages/english.md create mode 100644 docs/benchmarks/languages/finnish.md create mode 100644 docs/benchmarks/languages/french.md create mode 100644 docs/benchmarks/languages/german.md create mode 100644 docs/benchmarks/languages/hungarian.md create mode 100644 docs/benchmarks/languages/index.md create mode 100644 docs/benchmarks/languages/italian.md create mode 100644 docs/benchmarks/languages/norwegian-bokmal.md create mode 100644 docs/benchmarks/languages/norwegian-nynorsk.md create mode 100644 docs/benchmarks/languages/persian.md create mode 100644 docs/benchmarks/languages/polish.md create mode 100644 docs/benchmarks/languages/portuguese.md create mode 100644 docs/benchmarks/languages/russian.md create mode 100644 docs/benchmarks/languages/spanish.md create mode 100644 docs/benchmarks/languages/swedish.md create mode 100644 docs/benchmarks/languages/ukrainian.md create mode 100644 docs/benchmarks/languages/yiddish.md create mode 100644 docs/benchmarks/reference/candidates.md create mode 100644 docs/benchmarks/reference/corpora.md create mode 100644 docs/benchmarks/reference/english-coverage.md create mode 100644 docs/benchmarks/reference/environment.md create mode 100644 docs/benchmarks/reference/methodology.md create mode 100644 docs/fast-track.md create mode 100644 docs/integration-deep-dive.md create mode 100644 docs/migration-and-backward-compatibility.md create mode 100644 gradle/lucene-benchmarks.gradle create mode 100644 gradle/opennlp-benchmarks.gradle create mode 100644 gradle/paicehusk-benchmarks.gradle create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java create mode 100644 src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java create mode 100644 src/main/java/org/egothor/stemmer/CompiledPatchCommand.java create mode 100644 src/test/java/org/egothor/stemmer/CompiledPatchCommandTest.java create mode 100644 src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java create mode 100644 src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java create mode 100644 src/test/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpusTest.java create mode 100644 src/test/java/org/egothor/stemmer/benchmark/LucenePorterStemmerCopiedTest.java create mode 100644 src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0421e4d..a45fb6e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -56,7 +56,7 @@ jobs: test -f gradle/verification-metadata.xml - name: Run JMH benchmarks - run: ./gradlew clean jmh --no-daemon + run: ./gradlew clean jmh -Pjmh.includes='.*StemmerComparisonBenchmark.*' --no-daemon - name: Upload JMH reports uses: actions/upload-artifact@v4 @@ -65,4 +65,4 @@ jobs: path: | build/reports/jmh/** build/results/jmh/** - if-no-files-found: warn \ No newline at end of file + if-no-files-found: warn diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 51a66a0..8abb86e 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -70,7 +70,7 @@ jobs: test -f gradle/verification-metadata.xml - name: Build reports for publication - run: ./gradlew --no-daemon clean ciRelease pmdMain javadoc jacocoCiReleaseReport pitest jmh cyclonedxBom + run: ./gradlew --no-daemon clean ciRelease pmdMain javadoc jacocoCiReleaseReport pitest jmh -Pjmh.includes='.*StemmerComparisonBenchmark.*' cyclonedxBom - name: Prepare gh-pages worktree shell: bash @@ -191,11 +191,8 @@ jobs: COVERAGE_BADGE_LATEST_LINK='
  • Coverage Badge Metadata
    • ' MUTATION_BADGE_LINK='
  • Mutation Badge Metadata
    • ' MUTATION_BADGE_LATEST_LINK='
  • Mutation Badge Metadata
    • ' - JMH_BADGE_LINK='
  • Benchmark Badge Metadata
    • ' - JMH_BADGE_LATEST_LINK='
  • Benchmark Badge Metadata
    • ' COVERAGE_BADGE_REPORT_MD='- [Coverage badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json)' MUTATION_BADGE_REPORT_MD='- [Mutation badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json)' - JMH_BADGE_REPORT_MD='- [Benchmark badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/jmh-badge.json)' if [ ! -f "${RUN_METRICS_DIR}/coverage-badge.json" ]; then COVERAGE_BADGE_LINK='
  • Coverage Badge Metadata: not available
    • ' @@ -207,10 +204,6 @@ jobs: MUTATION_BADGE_REPORT_MD='- Mutation badge metadata: not currently available' fi - if [ ! -f "${RUN_METRICS_DIR}/jmh-badge.json" ]; then - JMH_BADGE_REPORT_MD='- Benchmark badge metadata: not currently available' - fi - cat > "${RUN_DIR}/index.html" < @@ -239,7 +232,6 @@ jobs: ${SBOM_XML_LINK:-
  • SBOM (XML): not available
    • } ${COVERAGE_BADGE_LINK} ${MUTATION_BADGE_LINK} - ${JMH_BADGE_LINK}
  • Mutation Testing Report
    • $( [ "${HAS_JMH}" = "true" ] && { echo "${JMH_TXT_LINK:-
  • Benchmark Results (TXT): not available
    • }"; echo "${JMH_CSV_LINK:-
  • Benchmark Results (CSV): not available
    • }"; } \ @@ -271,13 +263,12 @@ jobs: ${SBOM_JSON_REPORT_MD} ${SBOM_XML_REPORT_MD} - ## Benchmark and badge metadata + ## Benchmark reports and badge metadata ${JMH_TXT_REPORT_MD} ${JMH_CSV_REPORT_MD} ${COVERAGE_BADGE_REPORT_MD} ${MUTATION_BADGE_REPORT_MD} - ${JMH_BADGE_REPORT_MD} ## Historical runs diff --git a/README.md b/README.md index d654516..349e7e2 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ [![Quality gates](https://github.com/leogalambos/Radixor/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/leogalambos/Radixor/actions/workflows/build.yml) [![Coverage](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/coverage/) [![Mutation score](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/pitest/) -[![English benchmark](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/jmh-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt) -*Fast, deterministic, multi-language stemming for Java, built around compact patch-command tries and measured at roughly 4× to 6× the throughput of the Snowball Porter stemmer family on the current English benchmark workload.* +*Deterministic, multi-language stemming for Java, built around compact dictionary-derived patch-command tries with an explicit quality/speed trade-off.* **Radixor** is a modern multi-language stemming toolkit for Java in the tradition of the original **Egothor** approach. It learns compact word-to-stem transformations from dictionary data, stores them in compiled patch-command tries, and exposes a runtime model designed for speed, determinism, and operational simplicity. Unlike a closed-form dictionary lookup stemmer, Radixor can also generalize beyond explicitly listed word forms. @@ -50,18 +49,33 @@ Radixor is especially attractive when you want something more adaptable than sim ## Performance -Radixor includes a JMH benchmark suite for both its own algorithmic core and a side-by-side English comparison against the Snowball Porter stemmer family. +Radixor performance is best read together with stemming quality. The English dictionary coverage benchmark builds contracted compiled patch tries from deterministic slices of the `US_UK` dictionary and then measures both exact-root agreement and changed-token runtime. -On the current English comparison workload, Radixor with bundled `US_UK` reaches approximately **31 to 32 million tokens per second**. Snowball original Porter reaches approximately **8 million tokens per second**, and Snowball English (Porter2) approximately **5 to 5.5 million tokens per second**. +| Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 23.113 | 7.065 | 109.8 | +| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.270 | 9.914 | 101.0 | +| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 19.170 | 6.609 | 91.1 | +| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 20.857 | 6.734 | 99.1 | +| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 14.975 | 1.215 | 71.1 | +| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 15.249 | 1.078 | 72.4 | +| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.323 | 2.340 | 72.8 | +| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 16.778 | 2.643 | 79.7 | +| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 18.929 | 3.241 | 89.9 | +| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 19.124 | 1.883 | 90.9 | -That places Radixor at approximately: +Column meanings: -- **4× the throughput of Snowball original Porter** -- **6× the throughput of Snowball English (Porter2)** +- `Used rows` is the requested deterministic percentage of English dictionary rows used to build the stemmer. +- `Actual row ratio` is the selected row count divided by the full parsed dictionary row count. +- `All exact` is exact agreement over every word/root pair in the full dictionary. +- `Changed exact` is exact agreement only where the word differs from its root. +- `Root preserved` is the share of already-root forms that remain unchanged. +- `Speed ms/op` is JMH average time for one changed-token benchmark operation. +- `Error ms` is the JMH score error converted to milliseconds. +- `ns/token` is average nanoseconds per changed token in that operation. -on the current benchmark workload. - -This is a throughput comparison on the same deterministic token stream. It is **not** a claim that the compared stemmers are linguistically equivalent or interchangeable. +The contracted trie result is materially stronger than the older uncontracted profile: full English coverage reaches 97.478% all-token exactness and 97.197% changed-token exactness at 109.8 ns/token, while even a 10% deterministic dictionary slice remains at 92.868% all-token exactness and 76.516% changed-token exactness at 90.9 ns/token. This is why Radixor benchmark results are documented with both speed and quality instead of a single Porter speed badge. For benchmark scope, workload design, environment, commands, report locations, and interpretation guidance, see [Benchmarking](docs/benchmarking.md). @@ -101,6 +115,9 @@ Compared with the historical baseline, Radixor emphasizes: - **frequency-aware deterministic ordering** Candidate results are ordered consistently and reproducibly. +- **contracted compiled patch tries** + Uniform patch-command subtrees are collapsed into accepting leaves, reducing hot lookup depth while preserving preferred stemming results. + - **practical subtree reduction modes** Reduction can be tuned toward stronger compression or more conservative semantic preservation. @@ -131,8 +148,14 @@ The repository keeps the front page concise and places detailed documentation un ### Getting Started +- [Fast Track](docs/fast-track.md) + The shortest path from adding the dependency to getting a first stem from a bundled dictionary. + - [Quick Start](docs/quick-start.md) - A practical first guide to loading, compiling, and using Radixor. + A broader developer walkthrough covering loading options, querying, extension, persistence, and metadata. + +- [Integration Deep Dive](docs/integration-deep-dive.md) + Dependency setup, bundled dictionary selection, production lifecycle, search-pipeline guidance, and operational checklist. - [Built-in Languages](docs/built-in-languages.md) Overview of bundled language resources such as `US_UK`. @@ -157,6 +180,9 @@ The repository keeps the front page concise and places detailed documentation un - [Extending and Persisting Compiled Tries](docs/programmatic-extending-and-persistence.md) Reopening compiled tries, rebuilding them, and writing binary artifacts. +- [Migration and Backward Compatibility](docs/migration-and-backward-compatibility.md) + Migration from serialized String patch-command application to `CompiledPatchCommand`. + ### Concepts and Internals - [Architecture and Reduction Overview](docs/architecture-and-reduction.md) @@ -185,7 +211,10 @@ The repository keeps the front page concise and places detailed documentation un Engineering standards, validation posture, auditability, and operational model. - [Benchmarking](docs/benchmarking.md) - JMH benchmark methodology, Porter comparison, and result interpretation. + JMH benchmark methodology, dictionary coverage trade-offs, speed, quality, and result interpretation. + +- [Benchmark Results](docs/benchmarks/index.md) + Structured reference for methodology, corpora, environment, English coverage, and per-language result pages. - [Published Reports](docs/reports.md) Entry points to CI-published reports and GitHub Pages artifacts. diff --git a/build.gradle b/build.gradle index bbf08ea..fb93bf7 100644 --- a/build.gradle +++ b/build.gradle @@ -18,6 +18,8 @@ version = gitVersion(prefix:'release@') def benchmarkReportsDirectory = layout.buildDirectory.dir('reports/jmh') def sbomReportsDirectory = layout.buildDirectory.dir('reports/sbom') +def jmhIncludesProperty = providers.gradleProperty('jmh.includes') + .orElse(providers.systemProperty('jmh.includes')) def nvdApiKey = providers.gradleProperty('nvdApiKey') .orElse(providers.environmentVariable('NVD_API_KEY')) @@ -154,7 +156,10 @@ def configureJUnitPlatformTags = { Test task, String includeTagsExpr, String exc } tasks.named('test', Test) { - configureJUnitPlatformTags(it, cliIncludeTags, cliExcludeTags) + final def requestedIncludes = splitTagExpression(cliIncludeTags) + final boolean slowExplicitlyIncluded = requestedIncludes.contains('slow') + final String defaultExcludeTags = cliExcludeTags ?: (slowExplicitlyIncluded ? null : 'slow') + configureJUnitPlatformTags(it, cliIncludeTags, defaultExcludeTags) finalizedBy(tasks.named('jacocoTestReport')) } @@ -430,6 +435,9 @@ jmh { resultsFile = benchmarkReportsDirectory.map { it.file('jmh-results.csv').asFile }.get() humanOutputFile = benchmarkReportsDirectory.map { it.file('jmh-results.txt').asFile }.get() duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE + if (jmhIncludesProperty.isPresent()) { + includes = [jmhIncludesProperty.get()] + } } tasks.named('jmh') { @@ -437,6 +445,8 @@ tasks.named('jmh') { description = 'Runs JMH benchmarks for the Radixor algorithmic core and Snowball comparison suite.' } +apply from: 'gradle/lucene-benchmarks.gradle' + tasks.register('regressionArtifactGenerator', JavaExec) { group = 'verification' description = 'Generates deterministic compiled trie regression artifacts.' @@ -506,6 +516,8 @@ javadoc { } apply from: 'gradle/snowball-benchmarks.gradle' +apply from: 'gradle/paicehusk-benchmarks.gradle' +apply from: 'gradle/opennlp-benchmarks.gradle' gradle.taskGraph.whenReady { taskGraph -> def banner = """ diff --git a/docs/architecture.md b/docs/architecture.md index e672d96..7ae5866 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,6 +53,16 @@ Those patch-command values are inserted into a mutable trie keyed by the source Equivalent subtrees are merged into canonical reduced nodes. +Before a selected semantic reduction mode is applied, Radixor also performs uniform-subtree +contraction. If every reachable entry below a subtree resolves to the same preferred patch +command, that subtree can be represented as an accepting leaf for that command. Runtime lookup can +then stop at that leaf even when the input word still has remaining characters. + +This is a structural optimization of preferred-result lookup. It reduces trie depth in regions +where the remaining suffix cannot change the selected command, while preserving the `get()` result +used by the standard stemmer path. The benchmark tables in `docs/benchmarks/` are based on this +contracted compiled representation. + ### Compilation The reduced structure is frozen into an immutable compiled trie optimized for runtime lookup. @@ -124,7 +134,7 @@ At runtime, lookup is conceptually simple: 3. retrieve one or more stored patch commands, 4. apply the chosen patch command to the original word. -The trie itself does not create the final stem string. It selects the stored transformation command. `PatchCommandEncoder.apply(...)` then performs the actual transformation. +The trie itself does not create the final stem string. It selects the stored transformation command. Runtime code should use `CompiledPatchCommand.apply(...)` so the serialized command is compiled once and reused. That separation is architecturally important: @@ -170,6 +180,7 @@ The final compiled trie can be much smaller than the original dictionary for sev - patch commands are compact, - trie paths reuse shared structure, +- uniform preferred-command subtrees can be contracted into accepting leaves, - reduction merges equivalent subtrees, - binary persistence stores the already reduced form, - GZip compression is applied on top of the binary format. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 91bb8a9..9b6fa8c 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -1,159 +1,39 @@ # Benchmarking -Radixor includes a JMH benchmark suite for both the internal algorithmic core and a side-by-side English comparison against the Snowball Porter stemmer family. +Radixor contains internal trie microbenchmarks, a separate stemmer comparison suite, and a dictionary coverage benchmark for Radixor itself. Published stemmer comparison results must come only from benchmark classes matching `.*StemmerComparisonBenchmark.*`; internal `FrequencyTrie*` microbenchmarks are not part of those results. -This document explains what is benchmarked, how to run the suite, and how benchmark results should be interpreted. +This page is the entry point for benchmark interpretation. Detailed tables and long reference material are split into focused subpages so that important points do not get buried. -## Scope +## Key Takeaways -The benchmark suite currently covers two categories: +- Speed and accuracy must be read together. A faster row is not necessarily a better stemmer. +- Radixor is the quality-oriented baseline in same-language comparisons. Its exact-root accuracy is often close to 100%, while many faster competitors are light, minimal, possessive, or aggressive rule-based stemmers with much lower root agreement. +- The measured Radixor cost buys dictionary-trained stemming precision. That precision improves search quality by mapping inflected forms to intended dictionary roots instead of approximate or over-reduced stems. +- Speed benchmarks process changed dictionary tokens where the surface form differs from the expected root. Accuracy benchmarks process the complete dictionary. +- Accuracy-only benchmarks intentionally use one deterministic JMH measurement iteration without warmup because repeated precision passes would duplicate the same counters. +- The historical Porter performance badge is retired. Benchmark reporting now uses speed and quality tables rather than a single Porter ratio. -- Radixor core operations, -- English stemmer comparison on the same token workload. +## Benchmark Documentation Map -The comparison benchmark processes the same deterministic English token stream through: +| Page | Purpose | +| --- | --- | +| [Benchmark methodology](benchmarks/reference/methodology.md) | Workload design, speed pass, quality pass, normalization policy, and exact-root metrics. | +| [Benchmark corpora](benchmarks/reference/corpora.md) | Dictionary row counts, complete quality tokens, already-root tokens, changed speed tokens, and timing token counts. | +| [Benchmark environment and reports](benchmarks/reference/environment.md) | Hardware, OS, JVM, JMH settings, report files, and current badge/report policy. | +| [English dictionary coverage benchmark](benchmarks/reference/english-coverage.md) | The quality/speed operating curve for contracted Radixor tries built from 100% down to 10% of English dictionary rows. | +| [Candidate evaluation](benchmarks/reference/candidates.md) | Included benchmark families and evaluated candidates that were skipped. | +| [Language benchmark pages](benchmarks/languages/index.md) | Per-language accuracy tables, speed tables, and implementation notes. | -- Radixor with bundled `US_UK` (older benchmark snapshots used the now-retired `US_UK_PROFI` resource), -- Snowball original Porter, -- Snowball English, commonly referred to as Porter2. +## How To Read Results -The purpose of the comparison is throughput measurement on identical input. It is not intended to demonstrate linguistic equivalence between the compared stemmers. +Start with the [language benchmark pages](benchmarks/languages/index.md). Each language page lists accuracy first and speed second because throughput without root agreement is not enough to interpret stemmer quality. -## How to read the published numbers +When Radixor is slower than a narrow competitor, check the accuracy table before drawing a conclusion. Many Lucene light/minimal filters and possessive filters intentionally do less work. They can be fast precisely because they are not trying to match the dictionary root with the same precision. -Two kinds of benchmark numbers are relevant in the project. +The [English dictionary coverage benchmark](benchmarks/reference/english-coverage.md) shows the central operating curve explicitly: contracted tries preserve high quality even at reduced dictionary coverage, while changed-form exactness still reflects how much language knowledge was available during training. This is why Radixor performance should be discussed as a configurable quality/speed point, not as a single fixed ratio against Porter. -### Reference measurements +## Current Result Locations -The detailed benchmark snapshot documented on this page comes from a controlled run on a Ryzen 5 system. Those numbers are the best reference point for understanding absolute throughput under a known local benchmark environment. +The current measured language results are published in [Language Benchmark Pages](benchmarks/languages/index.md). Generated local report files for this benchmark update are listed in [Benchmark environment and reports](benchmarks/reference/environment.md). -### Published badge figures - -The benchmark badge metadata published through GitHub Pages is generated in the GitHub-hosted container environment. That environment is convenient for continuous publication, but it is not the right place to treat absolute throughput values as stable across time. CPU scheduling, shared-host variability, and container-level noise can materially affect raw numbers from run to run. - -For that reason, the published badge values should be treated primarily as a compact status surface. They are useful for observing broad trends and relative positioning, but not as the authoritative source for precise absolute throughput claims. - -## Current snapshot - -A recent JMH run on JDK 21.0.10 with JMH 1.37, one thread, three warmup iterations, and five measurement iterations produced the following approximate throughput ranges: - -| Workload | Radixor `US_UK` *(historical runs: `US_UK_PROFI`)* | Snowball Porter | Snowball English | -| --- | ---: | ---: | ---: | -| About 12,000 generated tokens | 30.99 M tokens/s | 8.21 M tokens/s | 5.46 M tokens/s | -| About 60,000 generated tokens | 32.25 M tokens/s | 8.02 M tokens/s | 5.11 M tokens/s | - -On that workload, Radixor measured approximately: - -- 4 times the throughput of Snowball original Porter, -- 6 times the throughput of Snowball English. - -These values are workload-dependent and environment-dependent. They should be read as measured results for the documented setup, not as universal constants. - -## Interpreting the relative result - -Although the absolute numbers can move across environments, the throughput relationship between Radixor and the compared Porter-family stemmers has remained broadly stable in practical measurements. In particular, the comparison against Snowball original Porter is consistently in the rough range of about four to one in Radixor’s favor. - -That relative behavior is more informative than any single absolute figure. It reflects a real architectural difference rather than a cosmetic benchmark artifact. - -Radixor is built around a compiled patch-command trie that resolves the result through a direct lookup and patch application path. In contrast, classic rule-based stemmers such as the Porter family follow a different operational model. The result is that Radixor combines two properties that do not often appear together: - -- dictionary-driven compiled lookup performance, -- the ability to generalize beyond explicitly listed word forms instead of behaving like a pure closed-form dictionary lookup table. - -Within that design space, the measured throughput profile is strong enough to place Radixor among the fastest known practical implementations of this kind, while still supporting stemming of previously unseen forms. That should still be read as a carefully bounded engineering statement, not as an absolute claim over every possible stemmer architecture or benchmark scenario. - -## Benchmark classes - -The main benchmark classes are under `src/jmh/java/org/egothor/stemmer/benchmark`. - -Relevant classes include: - -- `FrequencyTrieLookupBenchmark`, -- `FrequencyTrieCompilationBenchmark`, -- `EnglishStemmerComparisonBenchmark`. - -The English comparison benchmark uses the bundled Radixor English resource and the official Snowball Java distribution integrated into the JMH source set. - -## Workload design - -The English comparison benchmark uses a deterministic generated corpus rather than an uncontrolled ad hoc text sample. - -The workload intentionally mixes: - -- simple inflections, -- common derivational forms, -- US and UK spelling families, -- lexical forms appropriate for the current bundled `US_UK` resource (with historical continuity from earlier `US_UK_PROFI` runs). - -This design keeps runs reproducible across environments and avoids accidental drift caused by changing external corpora. - -## Running benchmarks - -Run the full benchmark suite: - -```bash -./gradlew jmh -``` - -Run only the English comparison benchmark: - -```bash -./gradlew jmh -Pjmh.includes=EnglishStemmerComparisonBenchmark -``` - -## Generated reports - -JMH reports are written to: - -- `build/reports/jmh/jmh-results.txt`, -- `build/reports/jmh/jmh-results.csv`. - -The text report is convenient for human review. The CSV report is more useful for CI archiving, historical tracking, and external processing. - -## Interpreting results responsibly - -Benchmark numbers should always be read with care. - -Important factors include: - -- CPU model and frequency behavior, -- thermal throttling, -- JVM vendor and version, -- system background load, -- operating-system scheduling noise, -- benchmark parameter changes. - -For meaningful comparison, keep these stable: - -- hardware or VM class, -- JDK version, -- benchmark parameters, -- thread count, -- benchmark source revision. - -If a regression is suspected, repeat the run and compare against previous CSV output rather than relying on a single measurement. - -## Regression tracking - -The recommended regression workflow is: - -1. archive `jmh-results.csv`, -2. compare the same benchmark names across runs, -3. compare only like-for-like environments, -4. investigate sustained regressions rather than one-off noise. - -For public reporting, the README should keep only the condensed benchmark summary, while detailed benchmark methodology and interpretation should remain in this document. - -## Notes on comparison fairness - -Radixor, Snowball Porter, and Snowball English are not the same kind of stemmer. - -Radixor uses a compiled patch-command trie driven by dictionary data. Snowball Porter and Snowball English are rule-based English stemmers. - -Because of that, the comparison should be understood as: - -- equal input workload, -- different stemming strategies, -- measured throughput rather than semantic identity. - -That distinction matters whenever performance claims are discussed in documentation, release notes, or badge summaries. +JMH TXT and CSV reports are still published as benchmark artifacts. They are no longer converted into a Shields endpoint benchmark badge. diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md new file mode 100644 index 0000000..0f43071 --- /dev/null +++ b/docs/benchmarks/index.md @@ -0,0 +1,55 @@ +# Benchmark Results + +This section contains the published Radixor benchmark reference set. It is intentionally split into +two layers: + +- **benchmark reference pages**, which explain methodology, corpora, environment, candidate + selection, and the English dictionary coverage experiment; +- **language result pages**, which contain the actual same-language accuracy and throughput tables. + +This structure keeps methodology separate from per-language result pages, while preserving all +measured data and the command-class analysis for each Radixor language resource. + +## Read This First + +Start with [Benchmarking](../benchmarking.md) for the high-level interpretation model. The most +important rule is that speed and exact-root quality must be read together. Many competing stemmers +are intentionally light, minimal, or aggressive; they can be fast because they are not trying to +match dictionary roots with the same precision. + +Radixor rows in the refreshed tables use contracted compiled patch tries. Contraction collapses +uniform preferred-command subtrees into accepting leaves, reducing hot lookup depth while preserving +the preferred result measured by the accuracy pass. + +## Reference Pages + +| Page | Purpose | +| --- | --- | +| [Methodology](reference/methodology.md) | Workload design, normalization, speed metrics, quality metrics, and interpretation rules. | +| [Corpora](reference/corpora.md) | Dictionary row counts, complete quality tokens, already-root tokens, changed speed tokens, and timing token counts. | +| [Environment and reports](reference/environment.md) | Hardware, JVM, JMH settings, report files, and badge/report policy. | +| [English dictionary coverage](reference/english-coverage.md) | Quality/speed operating curve for contracted Radixor tries built from 100% down to 10% of English dictionary rows. | +| [Candidate evaluation](reference/candidates.md) | Included benchmark families and evaluated candidates that were skipped. | + +## Language Results + +Each language page contains: + +- the dictionary corpus size, +- the Radixor patch-command distribution, +- exact-root quality metrics, +- throughput metrics, +- interpretation notes for the compared stemmers. + +Open [Language Benchmark Pages](languages/index.md) for the complete language list. + +## Key Published Result + +The English dictionary coverage benchmark shows the current contracted-trie operating curve. With +the full English dictionary, Radixor reaches `97.478%` all-token exactness and `97.197%` +changed-token exactness at `109.8 ns/token`. Even with a deterministic 10% dictionary slice, it +keeps `92.868%` all-token exactness and `76.516%` changed-token exactness at `90.9 ns/token`. + +Those figures should not be reduced to a single speed badge. The professional interpretation is a +quality/speed envelope: the amount and quality of dictionary knowledge affect stemming precision, +while contracted tries reduce lookup cost in uniform regions of the compiled graph. diff --git a/docs/benchmarks/languages/czech.md b/docs/benchmarks/languages/czech.md new file mode 100644 index 0000000..7eefeb5 --- /dev/null +++ b/docs/benchmarks/languages/czech.md @@ -0,0 +1,51 @@ +# Czech Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Czech. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `CS_CZ` | 5,113 | 56,612 | 10,049 | 46,563 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **56,612**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 675 | 1.192% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 22,681 | 40.064% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 14,980 | 26.461% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 10,109 | 17.857% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 8,167 | 14.426% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.465% | 99.439% | 99.582% | Full Radixor dictionary patch-command stemmer. | +| Lucene CzechStemFilter | 16.784% | 15.538% | 22.559% | Lucene Czech suffix stemmer implemented as a TokenFilter. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `czechRadixor` | 3.117 | 0.454 | 66.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene CzechStemFilter | `czechLuceneCzechStemFilter` | 2.921 | 0.202 | 62.7 | 0.937 | Czech suffix stemmer implemented as a Lucene TokenFilter. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/danish.md b/docs/benchmarks/languages/danish.md new file mode 100644 index 0000000..9d09561 --- /dev/null +++ b/docs/benchmarks/languages/danish.md @@ -0,0 +1,53 @@ +# Danish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Danish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `DA_DK` | 4,179 | 32,256 | 8,356 | 23,900 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **32,256**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 137 | 0.425% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 1,127 | 3.494% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 22,586 | 70.021% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 8,405 | 26.057% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 1 | 0.003% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.371% | 99.527% | 98.923% | Radixor baseline in the Snowball-language comparison family. | +| Lucene SnowballFilter | 55.509% | 54.159% | 59.371% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 55.509% | 54.159% | 59.371% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `radixor[DANISH]` | 1.065 | 0.019 | 44.6 | 1.000 | Radixor baseline for the Snowball-language comparison family. | +| Official Snowball direct | `snowballDirect[DANISH]` | 2.028 | 0.011 | 84.9 | 1.904 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[DANISH]` | 2.692 | 0.028 | 112.6 | 2.527 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/dutch.md b/docs/benchmarks/languages/dutch.md new file mode 100644 index 0000000..719a96d --- /dev/null +++ b/docs/benchmarks/languages/dutch.md @@ -0,0 +1,53 @@ +# Dutch Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Dutch. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `NL_NL` | 4,992 | 31,466 | 9,981 | 21,485 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **31,466**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 2,107 | 6.696% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 11,484 | 36.497% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 7,732 | 24.573% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 10,127 | 32.184% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 16 | 0.051% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.120% | 98.711% | 100.000% | Radixor baseline in the Snowball-language comparison family. | +| Official Snowball direct | 15.954% | 8.992% | 30.939% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene SnowballFilter | 12.620% | 5.441% | 28.073% | Lucene TokenFilter integration path around the Snowball algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `radixor[DUTCH]` | 1.262 | 0.039 | 58.7 | 1.000 | Radixor baseline for the Snowball-language comparison family. | +| Official Snowball direct | `snowballDirect[DUTCH]` | 3.968 | 0.258 | 184.7 | 3.145 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[DUTCH]` | 6.866 | 0.337 | 319.6 | 5.441 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/english.md b/docs/benchmarks/languages/english.md new file mode 100644 index 0000000..179cb08 --- /dev/null +++ b/docs/benchmarks/languages/english.md @@ -0,0 +1,67 @@ +# English Stemmer Benchmarks + +This page reports same-language stemming benchmarks for English. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `US_UK` | 396,939 | 1,004,374 | 793,874 | 210,500 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **1,004,374**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 28 | 0.003% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 22,493 | 2.240% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 186,764 | 18.595% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 795,024 | 79.156% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 65 | 0.006% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 97.478% | 97.197% | 97.552% | Full Radixor dictionary patch-command stemmer. | +| Lucene EnglishMinimalStemFilter | 90.981% | 65.189% | 97.820% | Minimal English plural reduction, not a full stemmer. | +| Lucene KStemFilter | 80.076% | 76.608% | 80.996% | Krovetz-style English stemming TokenFilter; broader than minimal suffix reducers. | +| Lucene EnglishPossessiveFilter | 79.032% | 0.003% | 99.987% | Possessive-ending remover only, not a full stemmer. | +| Snowball English / Porter2 | 40.342% | 46.296% | 38.763% | Porter2 rule-based suffix stemmer, distinct from original Porter. | +| Lucene PorterStemFilter | 39.538% | 46.201% | 37.772% | Lucene TokenFilter path for Porter suffix rules; not dictionary-root equivalent. | +| Lucene PorterStemmer direct copy | 39.538% | 46.201% | 37.772% | Direct Porter suffix-rule implementation generated under build for benchmark-only use. | +| OpenNLP PorterStemmer | 39.538% | 46.201% | 37.772% | Apache OpenNLP Porter suffix-rule implementation. | +| Snowball original Porter | 39.529% | 46.179% | 37.766% | Classic Porter rule-based suffix stemmer. | +| Paice/Husk Lancaster | 28.055% | 37.039% | 25.673% | Aggressive Paice/Husk rule stemmer that often produces shorter stems. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `radixorUsUkProfiPreferredStem` | 16.621 | 8.532 | 79.0 | 1.000 | Full dictionary patch-command stemmer using compiled patch commands. | +| Lucene EnglishPossessiveFilter | `luceneEnglishPossessiveFilter` | 23.845 | 0.833 | 113.3 | 1.435 | Possessive-ending remover only; not a full stemmer. | +| Lucene EnglishMinimalStemFilter | `luceneEnglishMinimalStemFilter` | 17.091 | 0.198 | 81.2 | 1.028 | Narrow plural reduction filter; not a full stemmer. | +| Lucene PorterStemmer direct copy | `lucenePorterStemmerCopied` | 18.598 | 10.954 | 88.4 | 1.119 | Benchmark-only generated copy of Lucene package-private Porter implementation. | +| OpenNLP PorterStemmer | `opennlpPorterStemmer` | 18.213 | 10.674 | 86.5 | 1.096 | Apache OpenNLP Porter implementation. | +| Snowball original Porter | `snowballOriginalPorter` | 32.921 | 11.520 | 156.4 | 1.981 | Classic Porter suffix-rule stemmer; historical English baseline, not a dictionary-equivalent stemmer. | +| Lucene PorterStemFilter | `lucenePorterStemFilter` | 42.874 | 1.321 | 203.7 | 2.579 | Lucene TokenFilter integration path for Porter; includes TokenStream overhead. | +| Lucene KStemFilter | `luceneKStemFilter` | 50.483 | 3.624 | 239.8 | 3.037 | Krovetz-style English TokenFilter; broader than minimal suffix filters. | +| Snowball English / Porter2 | `snowballEnglishPorter2` | 47.844 | 1.887 | 227.3 | 2.878 | Porter2 suffix-rule stemmer, distinct from original Porter. | +| Paice/Husk Lancaster | `paiceHuskLancaster` | 135.050 | 11.088 | 641.6 | 8.125 | Aggressive rule-based English stemmer. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/finnish.md b/docs/benchmarks/languages/finnish.md new file mode 100644 index 0000000..1435e88 --- /dev/null +++ b/docs/benchmarks/languages/finnish.md @@ -0,0 +1,55 @@ +# Finnish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Finnish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `FI_FI` | 57,027 | 1,865,215 | 110,525 | 1,754,690 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **1,865,215**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 745 | 0.040% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 1,176,003 | 63.049% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 565,585 | 30.323% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 116,946 | 6.270% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 5,936 | 0.318% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 98.661% | 98.803% | 96.408% | Full Radixor dictionary patch-command stemmer. | +| Lucene SnowballFilter | 10.991% | 10.268% | 22.471% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 10.991% | 10.268% | 22.471% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene FinnishLightStemFilter | 4.351% | 4.294% | 5.264% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `finnishRadixor` | 228.248 | 9.245 | 130.1 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene FinnishLightStemFilter | `finnishLuceneFinnishLightStemFilter` | 175.923 | 87.177 | 100.3 | 0.771 | Light Finnish suffix stemmer. | +| Official Snowball direct | `snowballDirect[FINNISH]` | 265.579 | 95.687 | 151.4 | 1.164 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[FINNISH]` | 338.099 | 175.101 | 192.7 | 1.481 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/french.md b/docs/benchmarks/languages/french.md new file mode 100644 index 0000000..b57b667 --- /dev/null +++ b/docs/benchmarks/languages/french.md @@ -0,0 +1,57 @@ +# French Stemmer Benchmarks + +This page reports same-language stemming benchmarks for French. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `FR_FR` | 59,240 | 474,110 | 108,141 | 365,969 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **474,110**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 5,370 | 1.133% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 185,263 | 39.076% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 153,886 | 32.458% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 116,519 | 24.576% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 13,072 | 2.757% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 94.831% | 94.859% | 94.734% | Full Radixor dictionary patch-command stemmer. | +| Lucene FrenchMinimalStemFilter | 11.472% | 6.236% | 29.192% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Lucene SnowballFilter | 8.551% | 5.183% | 19.952% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 8.462% | 5.067% | 19.952% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene FrenchLightStemFilter | 6.377% | 3.965% | 14.540% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `frenchRadixor` | 38.598 | 5.425 | 105.5 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene FrenchMinimalStemFilter | `frenchLuceneFrenchMinimalStemFilter` | 17.657 | 1.956 | 48.2 | 0.457 | Minimal French suffix reducer; narrow baseline. | +| Lucene FrenchLightStemFilter | `frenchLuceneFrenchLightStemFilter` | 28.742 | 2.391 | 78.5 | 0.745 | Light French suffix stemmer. | +| Official Snowball direct | `snowballDirect[FRENCH]` | 104.983 | 10.116 | 286.9 | 2.720 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[FRENCH]` | 117.938 | 4.007 | 322.3 | 3.056 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/german.md b/docs/benchmarks/languages/german.md new file mode 100644 index 0000000..548719e --- /dev/null +++ b/docs/benchmarks/languages/german.md @@ -0,0 +1,59 @@ +# German Stemmer Benchmarks + +This page reports same-language stemming benchmarks for German. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `DE_DE` | 39,315 | 213,440 | 73,799 | 139,641 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **213,440**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 3,627 | 1.699% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 48,605 | 22.772% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 80,443 | 37.689% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 75,717 | 35.475% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 5,048 | 2.365% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 97.455% | 97.973% | 96.476% | Full Radixor dictionary patch-command stemmer. | +| Lucene GermanLightStemFilter | 38.583% | 35.800% | 43.849% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene GermanMinimalStemFilter | 37.492% | 38.538% | 35.513% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Lucene SnowballFilter | 33.380% | 30.939% | 37.999% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 32.863% | 31.225% | 35.963% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene GermanStemFilter | 26.168% | 24.979% | 28.416% | German Lucene stemming TokenFilter; broader than minimal/light variants. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `germanRadixor` | 9.518 | 0.338 | 68.2 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene GermanMinimalStemFilter | `germanLuceneGermanMinimalStemFilter` | 12.020 | 0.460 | 86.1 | 1.263 | Minimal German suffix reduction; narrow baseline. | +| Lucene GermanLightStemFilter | `germanLuceneGermanLightStemFilter` | 12.413 | 1.059 | 88.9 | 1.304 | Light German suffix stemmer; narrower than a dictionary stemmer. | +| Lucene GermanStemFilter | `germanLuceneGermanStemFilter` | 36.644 | 5.667 | 262.4 | 3.850 | Older German stemming TokenFilter with normalization requirements. | +| Lucene SnowballFilter | `luceneSnowballFilter[GERMAN]` | 54.846 | 8.710 | 392.8 | 5.762 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | +| Official Snowball direct | `snowballDirect[GERMAN]` | 52.974 | 7.989 | 379.4 | 5.566 | Official Snowball generated Java stemmer; direct API. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/hungarian.md b/docs/benchmarks/languages/hungarian.md new file mode 100644 index 0000000..d26cb3f --- /dev/null +++ b/docs/benchmarks/languages/hungarian.md @@ -0,0 +1,55 @@ +# Hungarian Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Hungarian. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `HU_HU` | 19,406 | 935,713 | 38,775 | 896,938 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **935,713**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 15 | 0.002% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 149,173 | 15.942% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 746,296 | 79.757% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 40,125 | 4.288% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 104 | 0.011% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.222% | 99.537% | 91.948% | Full Radixor dictionary patch-command stemmer. | +| Lucene SnowballFilter | 66.445% | 66.938% | 55.043% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 66.445% | 66.938% | 55.043% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene HungarianLightStemFilter | 14.748% | 14.777% | 14.086% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `hungarianRadixor` | 53.844 | 5.619 | 60.0 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene HungarianLightStemFilter | `hungarianLuceneHungarianLightStemFilter` | 85.802 | 4.554 | 95.7 | 1.594 | Light Hungarian suffix stemmer. | +| Official Snowball direct | `snowballDirect[HUNGARIAN]` | 161.996 | 61.038 | 180.6 | 3.009 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[HUNGARIAN]` | 185.097 | 44.447 | 206.4 | 3.438 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/index.md b/docs/benchmarks/languages/index.md new file mode 100644 index 0000000..a1e1d2d --- /dev/null +++ b/docs/benchmarks/languages/index.md @@ -0,0 +1,45 @@ +# Language Benchmark Pages + +This section splits Radixor stemmer benchmark results by language. Each language page lists accuracy first and speed second. + +## Reference Pages + +| Page | Purpose | +| --- | --- | +| [Methodology](../reference/methodology.md) | Workload design, normalization, speed metrics, and quality metrics. | +| [Corpora](../reference/corpora.md) | Dictionary sizes and changed-token timing workloads. | +| [Environment and reports](../reference/environment.md) | Hardware, JVM, JMH settings, report files, and badge policy. | +| [English dictionary coverage](../reference/english-coverage.md) | Quality/speed operating curve for contracted Radixor tries built from 100% down to 10% of English dictionary rows. | +| [Candidate evaluation](../reference/candidates.md) | Included and skipped stemmer candidates. | + +## Languages + +| Language | Resource | Benchmark page | +| --- | --- | --- | +| Czech | `CS_CZ` | [Czech](czech.md) | +| Danish | `DA_DK` | [Danish](danish.md) | +| Dutch | `NL_NL` | [Dutch](dutch.md) | +| English | `US_UK` | [English](english.md) | +| Finnish | `FI_FI` | [Finnish](finnish.md) | +| French | `FR_FR` | [French](french.md) | +| German | `DE_DE` | [German](german.md) | +| Hungarian | `HU_HU` | [Hungarian](hungarian.md) | +| Italian | `IT_IT` | [Italian](italian.md) | +| Norwegian Bokmal | `NB_NO` | [Norwegian Bokmal](norwegian-bokmal.md) | +| Norwegian Nynorsk | `NN_NO` | [Norwegian Nynorsk](norwegian-nynorsk.md) | +| Persian | `FA_IR` | [Persian](persian.md) | +| Polish | `PL_PL` | [Polish](polish.md) | +| Portuguese | `PT_PT` | [Portuguese](portuguese.md) | +| Russian | `RU_RU` | [Russian](russian.md) | +| Spanish | `ES_ES` | [Spanish](spanish.md) | +| Swedish | `SV_SE` | [Swedish](swedish.md) | +| Ukrainian | `UK_UA` | [Ukrainian](ukrainian.md) | +| Yiddish | `YI` | [Yiddish](yiddish.md) | + +## Methodology Notes + +- Speed benchmarks process only changed dictionary tokens where the surface form differs from the expected root. +- Accuracy benchmarks process the complete dictionary and report `All exact`, `Changed exact`, and `Root preserved`. +- Radixor speed must be interpreted together with exact-root quality. A slower Radixor row must not be read as a simple performance weakness when Radixor is also the row with accuracy close to 100% and competing stemmers are much lower. Many fast light, minimal, possessive, or aggressive rule-based stemmers are fast because they do much less linguistic work. The measured Radixor cost buys dictionary-trained precision, and that precision is what improves search quality when queries and indexed text are reduced to the same intended roots. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows this contracted-trie operating curve explicitly. +- Results are comparable only within the same language and benchmark family. +- The historical Porter badge is retired; no JMH badge JSON is generated. diff --git a/docs/benchmarks/languages/italian.md b/docs/benchmarks/languages/italian.md new file mode 100644 index 0000000..060a376 --- /dev/null +++ b/docs/benchmarks/languages/italian.md @@ -0,0 +1,54 @@ +# Italian Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Italian. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `IT_IT` | 10,009 | 337,546 | 20,004 | 317,542 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **337,546**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 302,171 | 89.520% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 12,348 | 3.658% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 20,013 | 5.929% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 3,014 | 0.893% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.056% | 98.997% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Lucene ItalianLightStemFilter | 0.466% | 0.479% | 0.270% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene SnowballFilter | 0.041% | 0.043% | 0.010% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 0.041% | 0.043% | 0.010% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `italianRadixor` | 23.776 | 9.977 | 74.9 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene ItalianLightStemFilter | `italianLuceneItalianLightStemFilter` | 14.940 | 1.682 | 47.1 | 0.628 | Light Italian suffix stemmer. | +| Official Snowball direct | `snowballDirect[ITALIAN]` | 99.401 | 9.433 | 313.0 | 4.181 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[ITALIAN]` | 108.462 | 8.014 | 341.6 | 4.562 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/norwegian-bokmal.md b/docs/benchmarks/languages/norwegian-bokmal.md new file mode 100644 index 0000000..1221d8b --- /dev/null +++ b/docs/benchmarks/languages/norwegian-bokmal.md @@ -0,0 +1,57 @@ +# Norwegian Bokmal Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Norwegian Bokmal. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `NB_NO` | 17,929 | 90,757 | 33,376 | 57,381 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **90,757**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 1,500 | 1.653% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 4,296 | 4.734% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 47,619 | 52.469% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 34,420 | 37.925% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 2,922 | 3.220% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 96.852% | 97.637% | 95.503% | Full Radixor dictionary patch-command stemmer. | +| Lucene NorwegianMinimalStemFilter | 57.107% | 53.913% | 62.599% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Official Snowball direct | 54.824% | 51.791% | 60.040% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene SnowballFilter | 54.803% | 51.780% | 60.001% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Lucene NorwegianLightStemFilter | 52.136% | 50.616% | 54.749% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `norwegianBokmalRadixor` | 3.235 | 0.147 | 56.4 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene NorwegianMinimalStemFilter | `norwegianBokmalLuceneNorwegianMinimalStemFilter` | 2.720 | 0.190 | 47.4 | 0.841 | Minimal Norwegian suffix reducer. | +| Lucene NorwegianLightStemFilter | `norwegianBokmalLuceneNorwegianLightStemFilter` | 3.189 | 0.261 | 55.6 | 0.986 | Light Norwegian suffix stemmer. | +| Official Snowball direct | `snowballDirect[NORWEGIAN_BOKMAL]` | 3.978 | 0.031 | 69.3 | 1.230 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_BOKMAL]` | 5.526 | 0.409 | 96.3 | 1.708 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/norwegian-nynorsk.md b/docs/benchmarks/languages/norwegian-nynorsk.md new file mode 100644 index 0000000..914842a --- /dev/null +++ b/docs/benchmarks/languages/norwegian-nynorsk.md @@ -0,0 +1,53 @@ +# Norwegian Nynorsk Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Norwegian Nynorsk. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `NN_NO` | 4,688 | 19,651 | 6,089 | 13,562 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **19,651**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 224 | 1.140% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 1,505 | 7.659% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 11,017 | 56.063% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 6,427 | 32.706% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 478 | 2.432% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 93.089% | 91.395% | 96.863% | Radixor baseline in the Snowball-language comparison family. | +| Official Snowball direct | 60.974% | 60.212% | 62.670% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene SnowballFilter | 60.918% | 60.146% | 62.638% | Lucene TokenFilter integration path around the Snowball algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `radixor[NORWEGIAN_NYNORSK]` | 0.541 | 0.021 | 39.9 | 1.000 | Radixor baseline for the Snowball-language comparison family. | +| Official Snowball direct | `snowballDirect[NORWEGIAN_NYNORSK]` | 0.855 | 0.008 | 63.0 | 1.580 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[NORWEGIAN_NYNORSK]` | 1.221 | 0.025 | 90.0 | 2.258 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/persian.md b/docs/benchmarks/languages/persian.md new file mode 100644 index 0000000..13db80a --- /dev/null +++ b/docs/benchmarks/languages/persian.md @@ -0,0 +1,49 @@ +# Persian Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Persian. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `FA_IR` | 69 | 3,770 | 138 | 3,632 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **3,770**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `DeletePrefixCommand` | Deletes one or more leading characters from the word form in forward traversal. | 65 | 1.724% | +| `ForwardCompoundCommand` | Applies a multi-step forward patch made from skip, delete, insert, and replace operations. | 3,567 | 94.615% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 138 | 3.660% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 95.836% | 95.677% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Lucene PersianStemFilter | 1.485% | 0.000% | 40.580% | Lucene Persian suffix stemmer with required normalization in the measured path. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `persianRadixor` | 0.231 | 0.021 | 63.6 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene PersianStemFilter | `persianLucenePersianStemFilter` | 0.443 | 0.012 | 122.0 | 1.918 | Persian suffix stemmer with Lucene normalization in the measured path. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/polish.md b/docs/benchmarks/languages/polish.md new file mode 100644 index 0000000..5833b53 --- /dev/null +++ b/docs/benchmarks/languages/polish.md @@ -0,0 +1,55 @@ +# Polish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Polish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `PL_PL` | 9,990 | 132,308 | 19,957 | 112,351 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **132,308**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 1,719 | 1.299% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 53,303 | 40.287% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 37,051 | 28.004% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 20,415 | 15.430% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 19,820 | 14.980% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 98.837% | 98.744% | 99.359% | Full Radixor dictionary patch-command stemmer. | +| Lucene MorfologikFilter | 87.729% | 86.606% | 94.047% | Dictionary-based path; Morfologik can emit multiple terms. | +| Lucene StempelFilter | 70.009% | 69.262% | 74.220% | Lucene TokenFilter integration path for table-driven Polish Stempel. | +| Lucene StempelStemmer direct | 70.009% | 69.262% | 74.220% | Direct table-driven Polish Stempel stemmer API. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `polishRadixor` | 7.760 | 0.240 | 69.1 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene StempelStemmer direct | `polishLuceneStempelStemmerDirect` | 34.295 | 0.418 | 305.3 | 4.420 | Direct table-driven Polish Stempel stemmer API. | +| Lucene StempelFilter | `polishLuceneStempelFilter` | 39.116 | 1.717 | 348.2 | 5.041 | Lucene TokenFilter integration path for table-driven Polish Stempel. | +| Lucene MorfologikFilter | `polishLuceneMorfologikFilter` | 128.516 | 12.557 | 1143.9 | 16.562 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/portuguese.md b/docs/benchmarks/languages/portuguese.md new file mode 100644 index 0000000..1f4f3bb --- /dev/null +++ b/docs/benchmarks/languages/portuguese.md @@ -0,0 +1,59 @@ +# Portuguese Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Portuguese. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `PT_PT` | 4,001 | 215,490 | 8,002 | 207,488 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **215,490**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 3,806 | 1.766% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 120,691 | 56.008% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 71,284 | 33.080% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 8,003 | 3.714% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 11,706 | 5.432% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.815% | 99.808% | 100.000% | Full Radixor dictionary patch-command stemmer. | +| Lucene PortugueseLightStemFilter | 8.966% | 5.558% | 97.326% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene PortugueseMinimalStemFilter | 5.539% | 1.896% | 100.000% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Lucene SnowballFilter | 0.625% | 0.558% | 2.374% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 0.625% | 0.558% | 2.374% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene PortugueseStemFilter | 0.312% | 0.308% | 0.425% | Portuguese RSLP-style Lucene TokenFilter stemmer. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `portugueseRadixor` | 10.598 | 0.273 | 51.1 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene PortugueseLightStemFilter | `portugueseLucenePortugueseLightStemFilter` | 10.101 | 0.389 | 48.7 | 0.953 | Light Portuguese suffix stemmer. | +| Lucene PortugueseMinimalStemFilter | `portugueseLucenePortugueseMinimalStemFilter` | 14.493 | 0.760 | 69.8 | 1.367 | Minimal Portuguese suffix reducer. | +| Official Snowball direct | `snowballDirect[PORTUGUESE]` | 52.508 | 6.338 | 253.1 | 4.954 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[PORTUGUESE]` | 54.048 | 1.173 | 260.5 | 5.100 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | +| Lucene PortugueseStemFilter | `portugueseLucenePortugueseStemFilter` | 141.208 | 12.785 | 680.6 | 13.324 | Portuguese RSLP-style Lucene TokenFilter. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/russian.md b/docs/benchmarks/languages/russian.md new file mode 100644 index 0000000..4efe28b --- /dev/null +++ b/docs/benchmarks/languages/russian.md @@ -0,0 +1,55 @@ +# Russian Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Russian. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `RU_RU` | 37,410 | 806,279 | 74,808 | 731,471 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **806,279**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 9,260 | 1.148% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 584,785 | 72.529% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 82,864 | 10.277% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 75,646 | 9.382% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 53,724 | 6.663% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 98.807% | 98.696% | 99.896% | Full Radixor dictionary patch-command stemmer. | +| Lucene RussianLightStemFilter | 9.658% | 8.452% | 21.447% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene SnowballFilter | 9.162% | 8.162% | 18.936% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 9.162% | 8.162% | 18.936% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `russianRadixor` | 72.970 | 15.642 | 99.8 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene RussianLightStemFilter | `russianLuceneRussianLightStemFilter` | 57.900 | 4.404 | 79.2 | 0.793 | Light Russian suffix stemmer. | +| Official Snowball direct | `snowballDirect[RUSSIAN]` | 99.019 | 14.688 | 135.4 | 1.357 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[RUSSIAN]` | 128.272 | 7.201 | 175.4 | 1.758 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/spanish.md b/docs/benchmarks/languages/spanish.md new file mode 100644 index 0000000..27eb50b --- /dev/null +++ b/docs/benchmarks/languages/spanish.md @@ -0,0 +1,59 @@ +# Spanish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Spanish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `ES_ES` | 65,059 | 926,393 | 120,121 | 806,272 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **926,393**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 5,367 | 0.579% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 524,682 | 56.637% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 240,872 | 26.001% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 130,089 | 14.043% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 25,383 | 2.740% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 97.459% | 97.544% | 96.891% | Full Radixor dictionary patch-command stemmer. | +| Lucene SpanishMinimalStemFilter | 17.284% | 5.347% | 97.403% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Lucene SpanishPluralStemFilter | 15.140% | 5.802% | 77.820% | Plural-focused suffix reducer; narrow baseline. | +| Lucene SpanishLightStemFilter | 9.577% | 7.088% | 26.279% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Lucene SnowballFilter | 4.889% | 4.287% | 8.932% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 4.889% | 4.287% | 8.930% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `spanishRadixor` | 64.539 | 3.448 | 80.0 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene SpanishMinimalStemFilter | `spanishLuceneSpanishMinimalStemFilter` | 38.288 | 2.382 | 47.5 | 0.593 | Minimal Spanish suffix reducer; narrow baseline. | +| Lucene SpanishLightStemFilter | `spanishLuceneSpanishLightStemFilter` | 40.855 | 2.407 | 50.7 | 0.633 | Light Spanish suffix stemmer. | +| Lucene SpanishPluralStemFilter | `spanishLuceneSpanishPluralStemFilter` | 91.054 | 8.822 | 112.9 | 1.411 | Plural-oriented Spanish suffix reducer. | +| Official Snowball direct | `snowballDirect[SPANISH]` | 168.813 | 32.860 | 209.4 | 2.616 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[SPANISH]` | 185.626 | 50.297 | 230.2 | 2.876 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/swedish.md b/docs/benchmarks/languages/swedish.md new file mode 100644 index 0000000..ea55506 --- /dev/null +++ b/docs/benchmarks/languages/swedish.md @@ -0,0 +1,57 @@ +# Swedish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Swedish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `SV_SE` | 12,371 | 110,468 | 24,731 | 85,737 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **110,468**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 502 | 0.454% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 14,268 | 12.916% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 66,796 | 60.466% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 25,745 | 23.305% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 3,157 | 2.858% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 96.713% | 97.407% | 94.307% | Full Radixor dictionary patch-command stemmer. | +| Lucene SwedishMinimalStemFilter | 49.532% | 49.186% | 50.730% | Minimal suffix reducer; narrow baseline, not a full stemmer. | +| Lucene SwedishLightStemFilter | 45.672% | 46.383% | 43.209% | Light suffix stemmer; intentionally narrower than a dictionary-derived stemmer. | +| Official Snowball direct | 40.068% | 37.512% | 48.926% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | +| Lucene SnowballFilter | 38.785% | 35.839% | 48.999% | Lucene TokenFilter integration path around the Snowball algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `swedishRadixor` | 4.916 | 0.525 | 57.3 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Lucene SwedishMinimalStemFilter | `swedishLuceneSwedishMinimalStemFilter` | 4.453 | 0.530 | 51.9 | 0.906 | Minimal Swedish suffix reducer. | +| Lucene SwedishLightStemFilter | `swedishLuceneSwedishLightStemFilter` | 4.523 | 0.151 | 52.8 | 0.920 | Light Swedish suffix stemmer. | +| Official Snowball direct | `snowballDirect[SWEDISH]` | 7.075 | 0.541 | 82.5 | 1.439 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[SWEDISH]` | 9.379 | 0.056 | 109.4 | 1.908 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/ukrainian.md b/docs/benchmarks/languages/ukrainian.md new file mode 100644 index 0000000..b710ff4 --- /dev/null +++ b/docs/benchmarks/languages/ukrainian.md @@ -0,0 +1,53 @@ +# Ukrainian Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Ukrainian. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `UK_UA` | 1,493 | 15,737 | 2,985 | 12,752 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **15,737**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `AppendCharacterCommand` | Appends one character to the end of the word form. | 249 | 1.582% | +| `BackwardCompoundCommand` | Applies a multi-step backward patch made from skip, delete, insert, and replace operations. | 4,160 | 26.435% | +| `DeleteSuffixCommand` | Deletes one or more trailing characters from the word form. | 5,859 | 37.231% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 3,004 | 19.089% | +| `ReplaceLastCharacterCommand` | Replaces the final character of the word form. | 2,465 | 15.664% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 99.307% | 99.365% | 99.062% | Full Radixor dictionary patch-command stemmer. | +| Lucene MorfologikFilter | 92.362% | 90.637% | 99.732% | Dictionary-based path; Morfologik can emit multiple terms. | +| Morfologik direct | 92.362% | 90.637% | 99.732% | Direct dictionary lookup; first returned stem is used for quality when no ranking weight is exposed. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `ukrainianRadixor` | 0.605 | 0.056 | 47.4 | 1.000 | Full Radixor dictionary patch-command stemmer. | +| Morfologik direct | `ukrainianMorfologikDirect` | 8.106 | 0.040 | 635.7 | 13.408 | Direct Morfologik dictionary lookup; first returned stem is used for quality. | +| Lucene MorfologikFilter | `ukrainianLuceneMorfologikFilter` | 14.684 | 5.214 | 1151.5 | 24.287 | Dictionary-based Morfologik TokenFilter; may emit multiple terms. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/languages/yiddish.md b/docs/benchmarks/languages/yiddish.md new file mode 100644 index 0000000..5637e1c --- /dev/null +++ b/docs/benchmarks/languages/yiddish.md @@ -0,0 +1,52 @@ +# Yiddish Stemmer Benchmarks + +This page reports same-language stemming benchmarks for Yiddish. Accuracy is listed first because speed without root agreement is not enough to interpret stemmer quality. + +All speed values are environment-specific and were measured on the hardware and JVM listed in the [benchmark overview](../index.md). Speed benchmark operations process changed dictionary tokens only. Accuracy uses the complete Radixor dictionary for the language. + +Radixor must not be read as simply "slower" when a narrow competitor has a lower timing row. In these tables Radixor is the quality-oriented baseline: its exact-root accuracy is typically close to 100%, while many faster rule-based, light, minimal, or possessive filters reach that speed by doing much less linguistic work and often score far lower in `All exact` and `Changed exact`. The Radixor rows in this benchmark refresh use the contracted compiled patch trie: compilation collapses uniform patch-command subtrees into accepting leaves, reducing hot lookup depth while preserving the preferred stemming result measured by the accuracy pass. The [EnglishRadixorDictionaryCoverageBenchmark](../reference/english-coverage.md) table shows the resulting quality/speed envelope explicitly. The same interpretation applies to this language page: speed rows must be read together with the accuracy table above them. + +## Dictionary Corpus + +| Resource | Dictionary rows | Complete quality tokens | Already-root tokens | Changed speed tokens | +| --- | ---: | ---: | ---: | ---: | +| `YI` | 802 | 4,300 | 1,524 | 2,776 | + +## Radixor Patch Command Distribution + +Radixor stores the preferred transformation for each normalized dictionary word form as a compiled patch command. This distribution shows which runtime command class is selected by the trained trie for the complete language dictionary. The total number of preferred patch commands analyzed for this language is **4,300**. + +| Command class | Meaning | Word forms | Share | +| --- | --- | ---: | ---: | +| `DeletePrefixCommand` | Deletes one or more leading characters from the word form in forward traversal. | 25 | 0.581% | +| `ForwardCompoundCommand` | Applies a multi-step forward patch made from skip, delete, insert, and replace operations. | 2,721 | 63.279% | +| `PreserveCommand` | Returns the word form unchanged because it already matches the preferred root. | 1,551 | 36.070% | +| `ReplaceFirstCharacterCommand` | Replaces the first character of the word form in forward traversal. | 3 | 0.070% | + +## Accuracy + +Accuracy is computed from one deterministic JMH measurement iteration without warmup. The benchmark may execute the full dictionary pass more than once inside that single timed iteration; percentages divide matching counters by evaluated counters from the same iteration. + +| Stemmer | All exact | Changed exact | Root preserved | Note | +| --- | ---: | ---: | ---: | --- | +| Radixor | 98.930% | 98.343% | 100.000% | Radixor baseline in the Snowball-language comparison family. | +| Lucene SnowballFilter | 2.837% | 2.558% | 3.346% | Lucene TokenFilter integration path around the Snowball algorithm. | +| Official Snowball direct | 2.837% | 2.558% | 3.346% | Official Snowball generated Java stemmer; rule-based suffix algorithm. | + +## Speed + +Speed uses JMH average time, 3 warmup iterations, 5 measurement iterations, 1 fork, and 1 thread. Relative factor is computed against the single Radixor row on this language page. Values below 1.000 are faster than that Radixor baseline; values above 1.000 are slower. + +| Stemmer | Benchmark method | Score ms/op | Error ms | ns/token | Relative vs Radixor | Note | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Radixor | `radixor[YIDDISH]` | 0.236 | 0.004 | 85.1 | 1.000 | Radixor baseline for the Snowball-language comparison family. | +| Official Snowball direct | `snowballDirect[YIDDISH]` | 1.432 | 0.193 | 515.7 | 6.058 | Official Snowball generated Java stemmer; direct API. | +| Lucene SnowballFilter | `luceneSnowballFilter[YIDDISH]` | 1.595 | 0.068 | 574.6 | 6.749 | Lucene TokenFilter path around Snowball; includes TokenStream overhead. | + +## Interpretation Notes + +- Radixor is a dictionary-derived patch-command stemmer. Its quality depends on the language resource used to train the compiled trie. +- Light, minimal, plural, and possessive filters are narrow baselines. They can be fast because they intentionally perform less linguistic work. +- Lucene TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. +- Morfologik rows are dictionary-based and can emit multiple terms for one input token. Quality rows use the first returned term when no ranking weight is available. +- Snowball rows are rule-based generated suffix stemmers; they are useful algorithmic baselines, not dictionary-root equivalence guarantees. diff --git a/docs/benchmarks/reference/candidates.md b/docs/benchmarks/reference/candidates.md new file mode 100644 index 0000000..667d0ac --- /dev/null +++ b/docs/benchmarks/reference/candidates.md @@ -0,0 +1,23 @@ +# Benchmark Candidate Evaluation + +Implemented benchmark methods are documented on the per-language pages under [Language Benchmark Pages](../languages/index.md). This keeps the exact method names, implementation descriptions, accuracy tables, and speed tables close to the language where they are valid. + +## Included Candidate Families + +The current benchmark pages include Radixor baselines, Lucene language filters where the language matches a bundled Radixor resource, Lucene Stempel and Morfologik paths where applicable, official Snowball Java stemmers where same-language comparison is available, and selected English-specific non-Lucene baselines such as OpenNLP Porter and Paice/Husk Lancaster. + +Direct stemmer APIs and Lucene TokenFilter paths are documented separately on language pages. TokenFilter rows include TokenStream, attribute, and required normalization overhead. Direct rows measure exposed direct APIs. + +## Evaluated But Skipped Candidates + +| Candidate | Language | Link/source | Reason skipped | +| --- | --- | --- | --- | +| Lucene Arabic, Bulgarian, Bengali, Sorani, Greek, Galician, Hindi, Indonesian, Latvian, Telugu filters | Various | `lucene-analysis-common` | No bundled same-language Radixor resource in this repository snapshot. | +| Lucene analyzer-only paths | Multiple | Lucene analyzers | Full analyzers mix tokenization, stop-word handling, and other behavior; direct filters are used where available. | +| Lucene HunspellStemFilter | Multiple | `lucene-analysis-common` | Requires external Hunspell dictionaries not resolved as benchmark-only resources here. | +| Lucene StemmerOverrideFilter | Multiple | `lucene-analysis-common` | Override map facility, not a stemmer algorithm. | +| Additional Snowball Lovins | English | Official Snowball Java distribution | No Lovins Java stemmer was present in the selected Snowball Java distribution. | +| Lemur Project Krovetz Stemmer | English | Lemur Project | Lucene KStem represents the Krovetz-style path without adding separate dependency and license risk. | +| Smile Lancaster / Paice-Husk | English | Smile NLP | Smile is large for one stemmer; Paice/Husk is included through a smaller benchmark-only generated path. | +| CISTEM German stemmer | German | `https://github.com/LeonieWeissweiler/CISTEM` | Clean benchmark-only Java integration was not completed in this phase. | +| `stemmerEval` reference repository | Multiple | `https://github.com/endredy/stemmerEval` | Used only as a candidate reference; no code or data copied. | diff --git a/docs/benchmarks/reference/corpora.md b/docs/benchmarks/reference/corpora.md new file mode 100644 index 0000000..5226191 --- /dev/null +++ b/docs/benchmarks/reference/corpora.md @@ -0,0 +1,28 @@ +# Benchmark Corpora + +The table below describes the Radixor resources used to build speed and quality corpora. `Total tokens` is the complete dictionary token count used by quality benchmarks. `Already-root tokens` counts fields where the token is already equal to the line root. `Changed tokens` is the speed workload before the minimum-size repeat rule. + +| Language resource | Dictionary rows | Total tokens | Already-root tokens | Changed tokens | Speed timing tokens | +| --- | ---: | ---: | ---: | ---: | ---: | +| `cs_cz` | 5,113 | 56,612 | 10,049 | 46,563 | 46,563 | +| `da_dk` | 4,179 | 32,256 | 8,356 | 23,900 | 23,900 | +| `de_de` | 39,315 | 213,440 | 73,799 | 139,641 | 139,641 | +| `es_es` | 65,059 | 926,393 | 120,121 | 806,272 | 806,272 | +| `fa_ir` | 69 | 3,770 | 138 | 3,632 | 5,000 | +| `fi_fi` | 57,027 | 1,865,215 | 110,525 | 1,754,690 | 1,754,690 | +| `fr_fr` | 59,240 | 474,110 | 108,141 | 365,969 | 365,969 | +| `he_il` | 2,358 | 61,071 | 4,715 | 56,356 | 56,356 | +| `hu_hu` | 19,406 | 935,713 | 38,775 | 896,938 | 896,938 | +| `it_it` | 10,009 | 337,546 | 20,004 | 317,542 | 317,542 | +| `nb_no` | 17,929 | 90,757 | 33,376 | 57,381 | 57,381 | +| `nl_nl` | 4,992 | 31,466 | 9,981 | 21,485 | 21,485 | +| `nn_no` | 4,688 | 19,651 | 6,089 | 13,562 | 13,562 | +| `pl_pl` | 9,990 | 132,308 | 19,957 | 112,351 | 112,351 | +| `pt_pt` | 4,001 | 215,490 | 8,002 | 207,488 | 207,488 | +| `ru_ru` | 37,410 | 806,279 | 74,808 | 731,471 | 731,471 | +| `sv_se` | 12,371 | 110,468 | 24,731 | 85,737 | 85,737 | +| `uk_ua` | 1,493 | 15,737 | 2,985 | 12,752 | 12,752 | +| `us_uk` | 396,939 | 1,004,374 | 793,874 | 210,500 | 210,500 | +| `yi` | 802 | 4,300 | 1,524 | 2,776 | 5,000 | + +Speed benchmarks process the complete changed-token dictionary sequence for the language. Only resources with fewer than 5,000 changed tokens are repeated to reach the minimum timing size; larger resources are not sampled or truncated. diff --git a/docs/benchmarks/reference/english-coverage.md b/docs/benchmarks/reference/english-coverage.md new file mode 100644 index 0000000..ca8ffc2 --- /dev/null +++ b/docs/benchmarks/reference/english-coverage.md @@ -0,0 +1,43 @@ +# English Dictionary Coverage Benchmark + +`EnglishRadixorDictionaryCoverageBenchmark` builds Radixor from deterministic slices of the English dictionary rows and evaluates accuracy against the complete dictionary. The speed method then stems the full changed-token English timing corpus. + +This benchmark is the clearest demonstration of the Radixor quality/speed envelope after contracted-trie compilation. More dictionary knowledge still gives the strongest changed-form precision, but uniform-subtree contraction removes much of the historical lookup-depth penalty. The table should therefore be read as a measured operating curve rather than as a strictly monotonic function of dictionary size. + +| Used rows | Actual row ratio | All exact | Changed exact | Root preserved | Speed ms/op | Error ms | ns/token | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 100% | 100.000% | 97.478% | 97.197% | 97.552% | 23.113 | 7.065 | 109.8 | +| 90% | 90.000% | 97.047% | 94.913% | 97.613% | 21.270 | 9.914 | 101.0 | +| 80% | 80.000% | 96.635% | 92.768% | 97.661% | 19.170 | 6.609 | 91.1 | +| 70% | 70.000% | 96.209% | 90.565% | 97.705% | 20.857 | 6.734 | 99.1 | +| 60% | 60.000% | 95.750% | 88.384% | 97.703% | 14.975 | 1.215 | 71.1 | +| 50% | 50.000% | 95.262% | 86.107% | 97.690% | 15.249 | 1.078 | 72.4 | +| 40% | 40.000% | 94.753% | 83.855% | 97.643% | 15.323 | 2.340 | 72.8 | +| 30% | 30.000% | 94.208% | 81.651% | 97.537% | 16.778 | 2.643 | 79.7 | +| 20% | 20.000% | 93.633% | 79.366% | 97.416% | 18.929 | 3.241 | 89.9 | +| 10% | 10.000% | 92.868% | 76.516% | 97.204% | 19.124 | 1.883 | 90.9 | + +## Column Meanings + +- `Used rows`: requested deterministic percentage of English dictionary rows used to build the trie. +- `Actual row ratio`: selected rows divided by all parsed English dictionary rows. +- `All exact`: exact agreement over the complete dictionary. +- `Changed exact`: exact agreement over dictionary tokens where `token != expectedRoot`. +- `Root preserved`: percentage of already-root dictionary tokens that are left unchanged. +- `Speed ms/op`: JMH average time for one full changed-token English operation. +- `Error ms`: JMH score error converted to milliseconds. +- `ns/token`: `Speed ms/op` divided by 210,500 changed English tokens. + +For non-English languages, the same principle applies: dictionary-driven Radixor quality depends on the amount and consistency of the language resource, while contracted tries reduce the cost of uniform regions in the compiled lookup graph. The English table is the clearest because the English resource is large and the benchmark can show gradual deterministic reductions from 100% to 10%. + +## Why The Historical Porter Ratio Changed + +The historical English benchmark in `HEAD` used synthetic lexical families. Its `familyCount=5000` parameter generated roughly 70,000 artificial tokens rather than measuring the complete real English dictionary resource. That older workload was useful as a low-level stress test, but it was not a dictionary-quality comparison. Many synthetic tokens were not present in the Radixor dictionary, so Radixor often executed a fast miss path where lookup returned `null` and no patch command was applied. + +The current benchmark is intentionally based on real Radixor dictionary data. For English, the speed workload processes 210,500 changed token/root pairs where the dictionary token differs from the expected root, and the quality workload evaluates the complete 1,004,374-token dictionary. This is a hit-heavy workload that measures real lookup plus compiled patch-command application against known expected roots. It is therefore a different and more linguistically meaningful workload than the historical synthetic benchmark. + +The result must be interpreted in Radixor's favor through both speed and exact-root quality. Non-Radixor stemmers can look faster because many of them perform narrower or more aggressive transformations and do not attempt to match the dictionary root with the same precision. The English result table shows that this speed often comes with substantially lower `All exact` and `Changed exact` accuracy. + +Radixor uses the dictionary as training data for transformation rules. With the full English dictionary, it reaches much higher exact-root agreement than the Porter-family and other narrow baselines. Higher speed is still possible by reducing the amount or complexity of the input dictionary used to build the stemmer, but that is an explicit quality/speed trade-off rather than an accidental benchmark artifact. + +The coverage table shows that contracted tries substantially improve the operating point. Reducing dictionary knowledge still primarily damages changed-form exactness, while root preservation remains high. Even when Radixor is trained from only 10% of the English dictionary rows, the complete-dictionary `All exact` score remains above 92%. This is why Radixor performance should be discussed as a configurable quality/speed point, not as a single fixed ratio against Porter. diff --git a/docs/benchmarks/reference/environment.md b/docs/benchmarks/reference/environment.md new file mode 100644 index 0000000..97b0e77 --- /dev/null +++ b/docs/benchmarks/reference/environment.md @@ -0,0 +1,54 @@ +# Benchmark Environment And Reports + +The values below are environment-specific and must not be read as universal performance claims. + +| Item | Value | +| --- | --- | +| Benchmark date | 2026-07-03 | +| Focused comparison command family | JMH jar runs limited to `EnglishStemmerComparisonBenchmark`, `MultiLanguageStemmerComparisonBenchmark`, and `SnowballLanguageStemmerComparisonBenchmark`; Radixor exact-root metrics were recomputed deterministically against the same contracted loaders | +| English coverage command | `./gradlew jmh -Pjmh.includes='.*EnglishRadixorDictionaryCoverageBenchmark.*' --no-daemon` | +| Speed result reports | `build/reports/jmh/contracted/english-comparison.csv`, `multilanguage-speed.csv`, `snowball-language-speed.csv` | +| Accuracy result reports | Deterministic Radixor exact-root pass over bundled dictionaries; non-Radixor quality rows retained from the existing published quality suite | +| Final comparison JMH scope | Stemmer comparison benchmarks only; internal `FrequencyTrie*` microbenchmarks were not run | +| Coverage JMH scope | English Radixor dictionary coverage benchmark only | +| JMH version | 1.37 | +| Speed benchmark mode | Average time, `time/op` | +| Score unit | `ns/op` | +| Speed warmup | 3 iterations, 1 s each | +| Speed measurement | 5 iterations, 1 s each | +| Accuracy warmup | none for deterministic exact-root accounting | +| Accuracy measurement | 1 deterministic measurement iteration; counters only, not speed interpretation | +| Fork count in generated report files | 1 | +| Default fork policy for accuracy-only benchmark classes | `@Fork(0)` for future default runs because accuracy counters are deterministic and not interpreted as speed | +| Thread count | 1 | +| JVM reported by JMH | OpenJDK 64-Bit Server VM, 25.0.3+9 | +| JVM invoker | `/usr/lib/jvm/java-25-openjdk/bin/java` | +| Operating system | Linux 7.0.13-200.fc44.x86_64 | +| CPU | AMD Ryzen 5 7600 6-Core Processor | +| Logical CPUs | 12 | + +## Contracted Trie Baseline + +All Radixor rows in the refreshed benchmark tables use contracted compiled patch tries. During compilation, a subtree whose reachable entries all resolve to the same preferred patch command is represented as an accepting leaf. Runtime lookup can therefore stop as soon as that leaf is reached, which reduces depth in uniform regions while preserving the preferred result used by `get()`. + +## Report Files + +Generated local report files for this benchmark update: + +- `build/reports/jmh/contracted/english-comparison.csv` +- `build/reports/jmh/contracted/english-comparison.txt` +- `build/reports/jmh/contracted/multilanguage-speed.csv` +- `build/reports/jmh/contracted/multilanguage-speed.txt` +- `build/reports/jmh/contracted/snowball-language-speed.csv` +- `build/reports/jmh/contracted/snowball-language-speed.txt` + +JMH TXT and CSV reports are still published as benchmark artifacts. They are not converted into a Porter speed badge. + +## Published Metrics + +The historical English Radixor versus Porter performance badge is no longer generated. `tools/generate-pages-badges.py` now produces only coverage and mutation badge endpoint JSON files: + +- `coverage-badge.json` +- `pitest-badge.json` + +The README therefore no longer presents a single Porter speed ratio. Benchmark interpretation now uses both speed and quality, because a narrow or aggressive stemmer can be fast while disagreeing with the dictionary root much more often than Radixor. diff --git a/docs/benchmarks/reference/methodology.md b/docs/benchmarks/reference/methodology.md new file mode 100644 index 0000000..a92be84 --- /dev/null +++ b/docs/benchmarks/reference/methodology.md @@ -0,0 +1,59 @@ +# Benchmark Methodology + +The stemmer comparison suite measures Radixor and Java stemmers on the same language and deterministic Radixor dictionary-derived data. Published Radixor rows in this refresh use contracted compiled patch tries, where uniform preferred-command subtrees are collapsed into accepting leaves before the trie is frozen for lookup. For each language, the bundled dictionary resource stores the expected root as the first tab-separated field on a line and its surface forms on the same line. Every single-token field on that line can therefore be paired with the same expected root. + +Published stemmer comparison results must come only from benchmark classes matching `.*StemmerComparisonBenchmark.*`. Internal `FrequencyTrie*` microbenchmarks are not part of those results. + +## Benchmark Passes + +There are two distinct benchmark passes: + +- Speed benchmarks process only changed dictionary pairs where `token != expectedRoot`. This removes already-root tokens from timing so a stemmer is measured on words that actually require a transformation. If a language has fewer than 5,000 changed pairs, the complete changed-pair sequence is repeated in stable order until the timing corpus has at least 5,000 tokens. Larger changed-pair corpora are not sampled or truncated. +- Quality benchmarks process the complete dictionary for the language. They report exact agreement over all tokens, exact agreement over changed tokens only, and preservation of tokens that are already roots. + +Timing corpora are generated once per JMH JVM and kept in memory as shared `{token, expectedRoot}` arrays. Corpus construction, dictionary loading, trie loading, table loading, and analyzer construction are setup work and are not included in measured benchmark methods. + +Performance is interpreted as average time per input token: + +```text +timePerChangedTokenNs = JMH score ns/op / changedTimingTokenCount +``` + +This is necessary because Radixor dictionaries have different token counts by language. + +## Quality And Search Interpretation + +Radixor speed must be interpreted together with exact-root quality. A slower Radixor row must not be read as a simple performance weakness when Radixor is also the row with accuracy close to 100% and competing stemmers are much lower. + +Many fast light, minimal, possessive, or aggressive rule-based stemmers are fast because they do much less linguistic work. The measured Radixor cost buys dictionary-trained precision, and that precision is what improves search quality when queries and indexed text are reduced to the same intended roots. + +The [English dictionary coverage benchmark](english-coverage.md) shows this operating curve explicitly: contracted tries reduce lookup cost in uniform regions, while reduced dictionary coverage still lowers changed-form precision. + +## Normalization Policy + +Radixor is measured over dictionary tokens from its own resources: lower-case with `Locale.ROOT`, diacritics preserved. The corpus is normalized during setup, so the Radixor benchmark path uses `FrequencyTrie.getNormalized(CharSequence)` and does not measure redundant lookup-time lowercasing or diacritic normalization. + +Lucene TokenFilter paths include required normalization in the measured pipeline. Examples include lower-case normalization for filters requiring lower-case input, German normalization before German light/minimal stemming, and Persian decimal, Arabic, and Persian normalization before Persian stemming. No ASCII folding is applied to Czech or Polish paths, because those Lucene stemmers are diacritic-aware or dictionary/table-backed for those languages. TokenFilter throughput methods materialize each emitted `CharTermAttribute` as a `String` before passing it to the JMH `Blackhole`, so output consumption is easier to inspect and closer to the direct stemmer methods. + +For right-to-left Radixor languages, patch application uses the traversal direction stored in trie metadata. This is required because static backward patch application is not correct for all bundled languages. + +## Quality Metric + +The quality pass reports exact-root agreement against the expected root from the Radixor dictionary line. It writes to the normal JMH report files: + +- `build/reports/jmh/jmh-results.csv` +- `build/reports/jmh/jmh-results.txt` + +Accuracy is computed from standard JMH secondary rows: + +```text +allExactPercent = correctMatches / evaluatedTokens * 100 +changedExactPercent = changedCorrectMatches / changedEvaluatedTokens * 100 +rootPreservedPercent = rootPreservedMatches / rootEvaluatedTokens * 100 +``` + +`allExactPercent` uses the complete dictionary. `changedExactPercent` uses only tokens where `token != expectedRoot`. `rootPreservedPercent` measures whether a stemmer leaves already-root dictionary entries unchanged. + +Morfologik can emit multiple terms for one input token. The quality benchmark uses the first emitted term for exact-root accounting when no ranking weight is exposed. Throughput benchmarks for Morfologik TokenFilter paths consume all emitted terms. + +Quality reports intentionally use one deterministic measurement iteration without warmup, because exact-root agreement is not a timing metric and repeated precision passes would only duplicate the same counters. diff --git a/docs/built-in-languages.md b/docs/built-in-languages.md index 7a92fd5..0bed705 100644 --- a/docs/built-in-languages.md +++ b/docs/built-in-languages.md @@ -10,7 +10,7 @@ Bundled dictionaries are exposed through: org.egothor.stemmer.StemmerPatchTrieLoader.Language ``` -Each bundled dictionary is packaged with the library as a compressed UTF-8 text resource. When loaded, the resource is parsed by `StemmerDictionaryParser`, transformed into patch-command mappings, and compiled into a read-only `FrequencyTrie` by `StemmerPatchTrieLoader`. +Each bundled dictionary is packaged with the library as a compressed UTF-8 text resource. When loaded through the runtime API, the resource is parsed by `StemmerDictionaryParser`, transformed into patch-command mappings, and compiled into a read-only `FrequencyTrie` by `StemmerPatchTrieLoader`. The bundled language definition also carries a language-level right-to-left flag. That flag is used by the loader to derive the `WordTraversalDirection` used for both trie-key construction and patch-command generation. In practice, left-to-right bundled languages use historical backward Egothor traversal, while right-to-left bundled languages use forward traversal over the stored form. @@ -18,28 +18,28 @@ The bundled language definition also carries a language-level right-to-left flag The following bundled language identifiers are currently available: -| Language | Enum constant | Writing direction | Notes | -|---|---|---:|---| -| Czech | `CS_CZ` | LTR | Bundled general-purpose dictionary | -| Danish | `DA_DK` | LTR | Bundled general-purpose dictionary | -| German | `DE_DE` | LTR | Bundled general-purpose dictionary | -| Spanish | `ES_ES` | LTR | Bundled general-purpose dictionary | -| Persian | `FA_IR` | RTL | Bundled dictionary uses forward traversal over the stored form | -| Finnish | `FI_FI` | LTR | Bundled general-purpose dictionary | -| French | `FR_FR` | LTR | Bundled general-purpose dictionary | -| Hebrew | `HE_IL` | RTL | Bundled dictionary uses forward traversal over the stored form | -| Hungarian | `HU_HU` | LTR | Bundled general-purpose dictionary | -| Italian | `IT_IT` | LTR | Bundled general-purpose dictionary | -| Norwegian Bokmål | `NB_NO` | LTR | Bundled general-purpose dictionary | -| Dutch | `NL_NL` | LTR | Bundled general-purpose dictionary | -| Norwegian Nynorsk | `NN_NO` | LTR | Bundled general-purpose dictionary | -| Polish | `PL_PL` | LTR | Bundled general-purpose dictionary | -| Portuguese | `PT_PT` | LTR | Bundled general-purpose dictionary | -| Russian | `RU_RU` | LTR | Bundled general-purpose dictionary | -| Swedish | `SV_SE` | LTR | Bundled general-purpose dictionary | -| Ukrainian | `UK_UA` | LTR | Bundled general-purpose dictionary | -| English | `US_UK` | LTR | Bundled general-purpose dictionary | -| Yiddish | `YI` | RTL | Bundled dictionary uses forward traversal over the stored form | +| Language | Enum constant | Writing direction | Notes | Benchmark page | +|---|---|---:|---|---| +| Czech | `CS_CZ` | LTR | Bundled general-purpose dictionary | [Czech](benchmarks/languages/czech.md) | +| Danish | `DA_DK` | LTR | Bundled general-purpose dictionary | [Danish](benchmarks/languages/danish.md) | +| German | `DE_DE` | LTR | Bundled general-purpose dictionary | [German](benchmarks/languages/german.md) | +| Spanish | `ES_ES` | LTR | Bundled general-purpose dictionary | [Spanish](benchmarks/languages/spanish.md) | +| Persian | `FA_IR` | RTL | Bundled dictionary uses forward traversal over the stored form | [Persian](benchmarks/languages/persian.md) | +| Finnish | `FI_FI` | LTR | Bundled general-purpose dictionary | [Finnish](benchmarks/languages/finnish.md) | +| French | `FR_FR` | LTR | Bundled general-purpose dictionary | [French](benchmarks/languages/french.md) | +| Hebrew | `HE_IL` | RTL | Bundled dictionary uses forward traversal over the stored form | No same-language external benchmark in this run | +| Hungarian | `HU_HU` | LTR | Bundled general-purpose dictionary | [Hungarian](benchmarks/languages/hungarian.md) | +| Italian | `IT_IT` | LTR | Bundled general-purpose dictionary | [Italian](benchmarks/languages/italian.md) | +| Norwegian Bokmål | `NB_NO` | LTR | Bundled general-purpose dictionary | [Norwegian Bokmal](benchmarks/languages/norwegian-bokmal.md) | +| Dutch | `NL_NL` | LTR | Bundled general-purpose dictionary | [Dutch](benchmarks/languages/dutch.md) | +| Norwegian Nynorsk | `NN_NO` | LTR | Bundled general-purpose dictionary | [Norwegian Nynorsk](benchmarks/languages/norwegian-nynorsk.md) | +| Polish | `PL_PL` | LTR | Bundled general-purpose dictionary | [Polish](benchmarks/languages/polish.md) | +| Portuguese | `PT_PT` | LTR | Bundled general-purpose dictionary | [Portuguese](benchmarks/languages/portuguese.md) | +| Russian | `RU_RU` | LTR | Bundled general-purpose dictionary | [Russian](benchmarks/languages/russian.md) | +| Swedish | `SV_SE` | LTR | Bundled general-purpose dictionary | [Swedish](benchmarks/languages/swedish.md) | +| Ukrainian | `UK_UA` | LTR | Bundled general-purpose dictionary | [Ukrainian](benchmarks/languages/ukrainian.md) | +| English | `US_UK` | LTR | Bundled general-purpose dictionary | [English](benchmarks/languages/english.md) | +| Yiddish | `YI` | RTL | Bundled dictionary uses forward traversal over the stored form | [Yiddish](benchmarks/languages/yiddish.md) | ## Basic usage @@ -48,6 +48,7 @@ Load a bundled dictionary like this: ```java import java.io.IOException; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -59,7 +60,7 @@ public final class BuiltInExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( StemmerPatchTrieLoader.Language.US_UK, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); @@ -76,8 +77,8 @@ This call loads the bundled dictionary resource for the selected language, parse ```java import java.io.IOException; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; -import org.egothor.stemmer.PatchCommandEncoder; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -88,21 +89,21 @@ public final class EnglishExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( StemmerPatchTrieLoader.Language.US_UK, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); final String word = "running"; - final String patch = trie.get(word); - final String stem = PatchCommandEncoder.apply(word, patch, trie.traversalDirection()); + final CompiledPatchCommand patch = trie.get(word); + final String stem = patch == null ? word : patch.apply(word); System.out.println(word + " -> " + stem); } } ``` -Passing `trie.traversalDirection()` to `PatchCommandEncoder.apply(...)` is the correct general contract. It ensures that the patch is applied using the same logical traversal model that was used when the trie and its patch commands were produced. +`CompiledPatchCommand` values are compiled with the traversal direction used when the trie and its patch commands were produced. ## Traversal behavior and right-to-left languages @@ -114,7 +115,7 @@ For bundled right-to-left languages such as Persian, Hebrew, and Yiddish, Radixo - trie keys are traversed from the logical beginning of the stored form, - patch commands are generated in that same forward direction, -- patch application must use `WordTraversalDirection.FORWARD`, which is naturally obtained from `trie.traversalDirection()`. +- compiled patch-command application uses `WordTraversalDirection.FORWARD`, which is naturally captured when `loadCompiled(...)` creates `CompiledPatchCommand` values. This design keeps the traversal policy explicit and consistent across dictionary loading, trie lookup, binary persistence, builder reconstruction, and patch application. @@ -124,11 +125,16 @@ Bundled dictionaries can be compiled using any supported `ReductionMode`. The re Typical entry points are: -- `StemmerPatchTrieLoader.load(language, storeOriginal, reductionMode)` -- `StemmerPatchTrieLoader.load(language, storeOriginal, reductionSettings)` +- `StemmerPatchTrieLoader.loadCompiled(language, storeOriginal, reductionMode)` +- `StemmerPatchTrieLoader.loadCompiled(language, storeOriginal, reductionSettings)` For most users, `ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS` is the most conservative general-purpose choice because it preserves ranked `getAll(...)` behavior. +Compiled bundled dictionaries also use internal uniform-subtree contraction. If a whole subtree +would return the same preferred patch command, Radixor stores that subtree as an accepting leaf and +removes the deeper branches. This is the contracted trie representation used by the published +benchmark tables and is independent of the public reduction mode selected by the caller. + ## Intended role of bundled dictionaries Bundled dictionaries should be understood as practical default resources. @@ -178,7 +184,7 @@ For production systems, the most robust workflow is usually: 2. extend it with domain-specific forms if needed, 3. rebuild it into a binary artifact, 4. deploy that compiled binary artifact, -5. load it at runtime through `loadBinary(...)`. +5. load it at runtime through `loadBinaryCompiled(...)`. This avoids repeated startup parsing and makes the deployed stemming behavior explicit, reproducible, and versionable. @@ -190,6 +196,7 @@ import java.nio.file.Path; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.FrequencyTrieBuilders; +import org.egothor.stemmer.PatchCommandEncoder; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.ReductionSettings; import org.egothor.stemmer.StemmerPatchTrieBinaryIO; @@ -213,7 +220,11 @@ public final class BundledRefinementExample { ReductionSettings.withDefaults( ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); - builder.put("microservices", "Na"); + final PatchCommandEncoder encoder = PatchCommandEncoder.builder() + .traversalDirection(base.traversalDirection()) + .build(); + + builder.put("microservices", encoder.encode("microservices", "microservice")); final FrequencyTrie compiled = builder.build(); diff --git a/docs/cli-compilation.md b/docs/cli-compilation.md index f650f2e..04a4188 100644 --- a/docs/cli-compilation.md +++ b/docs/cli-compilation.md @@ -171,6 +171,11 @@ The short form `-h` is also supported. Reduction mode is not only a storage decision. It also influences what semantics are preserved when the mutable trie is compiled into its canonical read-only form. +Before the selected public reduction mode is applied, compilation performs uniform-subtree +contraction. If all reachable entries below a subtree select the same preferred patch command, the +compiler stores that subtree as an accepting leaf and removes the deeper branches. This reduces +runtime lookup depth without changing the preferred result returned by the standard stemming path. + ### Ranked `getAll()` equivalence `MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS` merges subtrees whose `getAll()` results remain equivalent for every reachable key suffix and whose local result ordering is the same. @@ -203,9 +208,9 @@ The CLI is best used as a preparation step during packaging, deployment, or cont ### Treat compiled files as versioned assets -A `.radixor.gz` file should be handled as a versioned output artifact. It represents a specific dictionary state, a specific reduction mode, and, where relevant, specific dominant-result thresholds. +A `.radixor.gz` file should be handled as a versioned output artifact. It represents a specific dictionary state, a specific reduction mode, whether uniform-subtree contraction was used, and, where relevant, specific dominant-result thresholds. -Compiled tries also persist a human-readable metadata block (`key=value` lines) that includes format version, traversal direction, RTL indicator, reduction mode, dominant thresholds, diacritic-processing mode, and case-processing mode. After decompression, you can inspect this block directly to identify what dictionary/trie configuration the artifact contains. The current CLI uses `DiacriticProcessingMode.AS_IS`; custom diacritic stripping is available through the programmatic builder and loader APIs rather than through a CLI flag. +Compiled tries also persist a human-readable metadata block (`key=value` lines) that includes format version, traversal direction, RTL indicator, reduction mode, contraction flag, dominant thresholds, diacritic-processing mode, and case-processing mode. After decompression, you can inspect this block directly to identify what dictionary/trie configuration the artifact contains. The current CLI uses `DiacriticProcessingMode.AS_IS`; custom diacritic stripping is available through the programmatic builder and loader APIs rather than through a CLI flag. ### Choose reduction mode deliberately @@ -237,11 +242,12 @@ java org.egothor.stemmer.Compile \ ### 3. Load it in an application ```java +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; -final FrequencyTrie trie = - StemmerPatchTrieLoader.loadBinary("english.radixor.gz"); +final FrequencyTrie trie = + StemmerPatchTrieLoader.loadBinaryCompiled("english.radixor.gz"); ``` ## Exit codes and error handling diff --git a/docs/compatibility-and-guarantees.md b/docs/compatibility-and-guarantees.md index fdf32c6..d64af19 100644 --- a/docs/compatibility-and-guarantees.md +++ b/docs/compatibility-and-guarantees.md @@ -81,7 +81,11 @@ Compiled `FrequencyTrie` instances are immutable and thread-safe for concurrent ### Stable patch application behavior -`PatchCommandEncoder.apply(...)` remains the compatibility API for string-returning patch application. Buffer-oriented `applyTo(...)` overloads are additive APIs for caller-owned output storage. They do not retain output arrays, report insufficient capacity with `APPLY_INSUFFICIENT_CAPACITY`, and preserve the existing malformed-patch compatibility behavior where `apply(...)` preserves the source. +Serialized patch-command strings remain the stable stored representation used by textual dictionaries and binary artifacts. Runtime stemming should use `CompiledPatchCommand` values produced by `StemmerPatchTrieLoader.loadCompiled(...)`, `StemmerPatchTrieLoader.loadBinaryCompiled(...)`, or `PatchCommandEncoder.compile(...)`. + +The historical `PatchCommandEncoder.apply(...)` and String-based `applyTo(...)` overloads remain compatibility APIs during the 2.x transition, but they are deprecated because they reparse the patch-command string on each application. See [Migration and Backward Compatibility](migration-and-backward-compatibility.md) for old and new code examples. + +Compiled buffer-oriented `CompiledPatchCommand.applyTo(...)` overloads use caller-owned output storage. They do not retain output arrays and report insufficient capacity with `CompiledPatchCommand.APPLY_INSUFFICIENT_CAPACITY`. ### Stable reduction-mode intent @@ -89,6 +93,11 @@ Each public `ReductionMode` constant carries a semantic contract that should rem In other words, the implementation may evolve, but the intended meaning of modes such as ranked `getAll()` equivalence, unordered `getAll()` equivalence, and dominant `get()` equivalence should not drift casually. +Internal pre-reduction optimizations may still change the physical compiled trie shape when they +preserve the documented lookup contract. Uniform-subtree contraction is one such optimization: it +can replace a subtree with an accepting leaf when all reachable entries choose the same preferred +patch command. + ### Stable binary artifact purpose Compiled `.radixor.gz` artifacts are a first-class project output. Loading and persisting compiled stemmer artifacts is part of the intended usage model, not an incidental implementation side effect. diff --git a/docs/fast-track.md b/docs/fast-track.md new file mode 100644 index 0000000..e17f1f6 --- /dev/null +++ b/docs/fast-track.md @@ -0,0 +1,130 @@ +# Fast Track + +This page is the shortest path from an empty Java project to a working Radixor stemmer. +It deliberately uses a bundled dictionary and the preferred compiled-command runtime API, so the +first result does not require writing a dictionary, running the CLI compiler, or understanding +reduction internals. + +Use this page when the goal is: + +- add the dependency, +- load a bundled language resource, +- stem a token, +- know where to go next. + +For deeper production guidance, see [Integration Deep Dive](integration-deep-dive.md). + +## 1. Add The Dependency + +Radixor is published as: + +```text +groupId: org.egothor +artifactId: radixor +``` + +Use the current published version from Maven Central. The snippets below use `3.0.0`; replace it +with the version you deploy if a newer release is available. + +For a Gradle project: + +```kotlin +dependencies { + implementation("org.egothor:radixor:3.0.0") +} +``` + +For a Maven project: + +```xml + + org.egothor + radixor + 3.0.0 + +``` + +Radixor targets modern Java and has a dependency-light runtime core. The project documentation and +benchmarks assume a current JDK; Java 21 or newer is the practical baseline for current releases. + +## 2. Load A Bundled Dictionary + +The fastest path is to use a bundled dictionary through `StemmerPatchTrieLoader.Language`. +This example uses the bundled English resource, `US_UK`. + +```java +import java.io.IOException; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; + +public final class RadixorFirstStem { + + private RadixorFirstStem() { + throw new AssertionError("No instances."); + } + + public static void main(final String[] arguments) throws IOException { + final FrequencyTrie stemmer = StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, + true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); + + final String token = "running"; + final CompiledPatchCommand command = stemmer.get(token); + final String stem = command == null ? token : command.apply(token); + + System.out.println(token + " -> " + stem); + } +} +``` + +The loaded `FrequencyTrie` is immutable and can be shared across request +threads. Load it once during application startup and reuse it for indexing and query processing. + +## 3. Choose A Language Resource + +Bundled dictionaries are exposed as enum constants. Common examples: + +| Language | Enum constant | +| --- | --- | +| English | `US_UK` | +| German | `DE_DE` | +| French | `FR_FR` | +| Spanish | `ES_ES` | +| Italian | `IT_IT` | +| Polish | `PL_PL` | +| Russian | `RU_RU` | +| Czech | `CS_CZ` | + +The full list, writing-direction notes, and benchmark links are in +[Built-in Languages](built-in-languages.md). + +## 4. Use The Same Stemmer On Both Sides + +For search, use the same Radixor configuration during indexing and query processing. A typical +minimal integration flow is: + +1. tokenize text with your application or search platform, +2. normalize tokens consistently, +3. call `stemmer.get(token)`, +4. apply the returned `CompiledPatchCommand`, +5. index or query with the resulting stem. + +Do not load the trie per token. The compiled trie is the runtime artifact; per-token work should be +limited to lookup and patch application. + +## 5. Next Step For Production + +The fast path compiles a bundled dictionary during startup. That is convenient for evaluation and +small services. For larger deployments, compile once, persist a `.radixor.gz` artifact, and load +that binary artifact at runtime. + +Continue with: + +- [Integration Deep Dive](integration-deep-dive.md) for production lifecycle guidance. +- [Loading and Building Stemmers](programmatic-loading-and-building.md) for all loading APIs. +- [Built-in Languages](built-in-languages.md) for bundled resources and dictionary locations. +- [Benchmarking](benchmarking.md) for speed and quality interpretation. diff --git a/docs/index.md b/docs/index.md index ee2b40a..2388f2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,8 +30,10 @@ Radixor is intended for teams that require consistent stemming quality at scale, ## Start here -- Read [Quick Start](quick-start.md) for immediate implementation guidance. -- Use [Programmatic Usage](programmatic-usage.md) for application integration patterns. -- Review [Benchmarking](benchmarking.md) for reproducible performance methodology. +- Read [Fast Track](fast-track.md) when you want the shortest path to a working bundled stemmer. +- Use [Integration Deep Dive](integration-deep-dive.md) when you are wiring Radixor into a real application or search pipeline. +- Read [Quick Start](quick-start.md) for the broader developer walkthrough after the first result works. +- Use [Built-in Languages](built-in-languages.md) to find the bundled dictionaries exposed by Radixor. +- Review [Benchmarking](benchmarking.md) and [Benchmark Results](benchmarks/index.md) for reproducible performance and quality methodology. - Open [CI Reports](reports.md) to inspect published build artifacts and quality metrics. - See the historical paper: [*Lemmatizer for Document Information Retrieval Systems in JAVA*](https://www.researchgate.net/publication/221512865_Lemmatizer_for_Document_Information_Retrieval_Systems_in_JAVA). diff --git a/docs/integration-deep-dive.md b/docs/integration-deep-dive.md new file mode 100644 index 0000000..71aa6cb --- /dev/null +++ b/docs/integration-deep-dive.md @@ -0,0 +1,235 @@ +# Integration Deep Dive + +This page explains how to integrate Radixor into a real Java application after the first +fast-track experiment works. It covers dependencies, bundled dictionaries, runtime lifecycle, +deployment artifacts, and the decisions that matter in search or text-processing systems. + +## Integration Model + +Radixor has two separate phases: + +| Phase | Work | Typical location | +| --- | --- | --- | +| Preparation | Parse dictionaries, derive patch commands, reduce and contract the trie, optionally persist a binary artifact. | Build pipeline, packaging job, admin tool, or startup for small services. | +| Runtime | Load an immutable compiled trie, look up patch commands, apply them to tokens. | Search indexing, query processing, text normalization, enrichment pipelines. | + +The practical rule is simple: compile rarely, stem often. + +For production systems, prefer a startup-owned or dependency-injected singleton +`FrequencyTrie` per language/configuration. The trie is immutable after +construction and is suitable for concurrent reads. + +## Dependency Coordinates + +The Maven coordinates are: + +```text +org.egothor:radixor +``` + +Gradle: + +```kotlin +dependencies { + implementation("org.egothor:radixor:3.0.0") +} +``` + +Maven: + +```xml + + org.egothor + radixor + 3.0.0 + +``` + +Replace `3.0.0` with the current release selected for your deployment. + +The core Java module is: + +```java +module org.egothor.radixor; +``` + +A named consuming module declares: + +```java +module example.search { + requires org.egothor.radixor; +} +``` + +## Bundled Dictionaries + +Radixor ships bundled dictionaries inside the library artifact. The public API exposes them through: + +```java +StemmerPatchTrieLoader.Language +``` + +The physical resources are packaged as compressed UTF-8 dictionaries under resource directories +such as: + +```text +us_uk/stemmer.gz +de_de/stemmer.gz +fr_fr/stemmer.gz +pl_pl/stemmer.gz +``` + +Treat those resource paths as implementation details. Application code should load bundled +dictionaries through `StemmerPatchTrieLoader.Language`, because the enum also carries the language +metadata needed for correct traversal. + +See [Built-in Languages](built-in-languages.md) for the complete language list, writing-direction +notes, and links to per-language benchmark pages. + +## Minimal Service Wrapper + +A small service wrapper keeps loading, null handling, and fallback behavior in one place. + +```java +import java.io.IOException; +import java.util.Objects; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; + +public final class RadixorStemmerService { + + private final FrequencyTrie trie; + + public RadixorStemmerService(final StemmerPatchTrieLoader.Language language) throws IOException { + this.trie = StemmerPatchTrieLoader.loadCompiled( + Objects.requireNonNull(language, "language"), + true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); + } + + public String stem(final String token) { + final String checkedToken = Objects.requireNonNull(token, "token"); + final CompiledPatchCommand command = trie.get(checkedToken); + return command == null ? checkedToken : command.apply(checkedToken); + } +} +``` + +The fallback behavior preserves the original token when the trie has no patch command for it. That +is usually the right default for search normalization, because unknown tokens should remain +searchable. + +## Production Artifact Workflow + +For a controlled deployment, compile once and deploy the binary artifact: + +1. choose a bundled or custom dictionary, +2. optionally extend it with domain vocabulary, +3. compile a contracted trie, +4. persist it as `.radixor.gz`, +5. deploy that artifact with the application, +6. load it with `StemmerPatchTrieLoader.loadBinaryCompiled(...)`. + +Runtime loading then avoids dictionary parsing and preparation-time memory pressure. + +```java +import java.io.IOException; +import java.nio.file.Path; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.StemmerPatchTrieLoader; + +public final class BinaryStemmerLoader { + + private BinaryStemmerLoader() { + throw new AssertionError("No instances."); + } + + public static FrequencyTrie loadEnglish() throws IOException { + return StemmerPatchTrieLoader.loadBinaryCompiled(Path.of("stemmers", "english.radixor.gz")); + } +} +``` + +Use [CLI Compilation](cli-compilation.md) for command-line artifact creation, or +[Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) for +programmatic artifact generation. + +## Search Pipeline Guidance + +Use Radixor consistently across indexing and querying: + +- choose one language dictionary per field or per analysis chain, +- apply the same token normalization before stemming on both sides, +- keep the compiled trie in memory and reuse it, +- use `get(...)` for a single preferred stem, +- use `getAll(...)` when a retrieval model benefits from preserving alternatives, +- version custom `.radixor.gz` artifacts with the application or index schema. + +For multilingual content, do not run every token through every language. Route text by field, +document metadata, or language detection before stemming. + +## Choosing Bundled Versus Custom Dictionaries + +Start with bundled dictionaries when: + +- the language is supported, +- the application needs a strong baseline quickly, +- the vocabulary is general-purpose, +- the team is evaluating Radixor or building an initial integration. + +Use custom or extended dictionaries when: + +- domain vocabulary changes search quality, +- product names, technical terms, legal terms, or biomedical terms must be preserved consistently, +- stemming behavior must be curated and reviewed, +- a release process needs a versioned lexical artifact. + +The dictionary format is intentionally simple and documented in +[Dictionary Format](dictionary-format.md). Contribution standards are described in +[Contributing Dictionaries](contributing-dictionaries.md). + +## Performance Practices + +The hot path should be only: + +```text +token -> trie lookup -> compiled command application -> stem +``` + +Avoid these patterns in production request paths: + +- loading or compiling dictionaries per request, +- applying serialized patch strings repeatedly instead of `CompiledPatchCommand`, +- rebuilding tries for short-lived batches, +- mixing different stemmer configurations between indexing and querying, +- interpreting speed without checking exact-root quality. + +The current benchmark documentation separates methodology, corpora, environment, and language +results so performance claims remain auditable. Start with [Benchmarking](benchmarking.md), then +use [Benchmark Results](benchmarks/index.md) for the detailed reference tree. + +## Operational Checklist + +Before production rollout: + +- dependency version is pinned, +- language resource and reduction mode are documented, +- indexing and query pipelines use the same stemming configuration, +- custom artifacts are versioned and reproducible, +- fallback behavior for unknown tokens is explicit, +- benchmark expectations are read together with quality metrics, +- CI includes at least a smoke test that stems representative project vocabulary. + +## Related Pages + +- [Fast Track](fast-track.md) +- [Quick Start](quick-start.md) +- [Built-in Languages](built-in-languages.md) +- [Programmatic Usage](programmatic-usage.md) +- [CLI Compilation](cli-compilation.md) +- [Benchmarking](benchmarking.md) diff --git a/docs/lookup-edge-optimization.md b/docs/lookup-edge-optimization.md index f98e8c3..88bfaf5 100644 --- a/docs/lookup-edge-optimization.md +++ b/docs/lookup-edge-optimization.md @@ -6,7 +6,10 @@ Compiled trie nodes (`CompiledNode`) use three lookup strategies when resolving 2. linear scan for very small child counts, 3. binary search over sorted edge labels. -This page explains the dense path, what `maxExpandedIndex` controls, and how to tune it. +This page explains the dense path, what `maxExpandedIndex` controls, and how to tune it. These +edge lookup strategies operate after trie reduction and uniform-subtree contraction. If lookup +reaches an accepting contracted leaf, no child edge search is needed for the remaining input +characters. ## Runtime model of one node @@ -75,7 +78,7 @@ The value never changes lookup semantics. It only changes the in-memory structur This threshold is **not** stored in `TrieMetadata`. - The binary format stores only trie payload and semantic metadata (`reduction`, `traversal`, - case/diacritic settings, and stream version). + case/diacritic settings, contraction settings, and stream version). - `maxExpandedIndex` is chosen when materializing nodes in memory. - You can therefore keep one persisted artifact and load it with different in-memory trade-offs depending on deployment constraints. @@ -135,15 +138,17 @@ At artifact load time, you can tune the same trade-off independently of persiste ```java import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; -var defaultLookup = StemmerPatchTrieLoader.loadBinary( +final FrequencyTrie defaultLookup = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz")); -var fastLookup = StemmerPatchTrieLoader.loadBinary( +final FrequencyTrie fastLookup = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 1024); -var compactLookup = StemmerPatchTrieLoader.loadBinary( +final FrequencyTrie compactLookup = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 0); ``` diff --git a/docs/migration-and-backward-compatibility.md b/docs/migration-and-backward-compatibility.md new file mode 100644 index 0000000..b5f514b --- /dev/null +++ b/docs/migration-and-backward-compatibility.md @@ -0,0 +1,155 @@ +# Migration and Backward Compatibility + +This page describes the migration from repeated serialized patch-command application to compiled patch commands. + +## Summary + +Radixor patch commands are still encoded as compact strings when dictionaries are built and persisted. That serialized form remains the interchange format used by textual dictionaries, binary artifacts, and compilation tooling. + +Runtime stemming should no longer repeatedly apply those serialized strings directly. Since 2.3.0, the String-based patch application API is deprecated. Code that stems live input should load or create `CompiledPatchCommand` values and reuse them. The deprecated API remains available for compatibility during the transition, but applications should migrate before 3.0.0. + +The reason is performance. The old API parses the serialized P-command every time it is applied. `CompiledPatchCommand` parses it once and stores a concrete immutable command object, so repeated stemming avoids the same analysis work. + +## Deprecated Runtime APIs + +The following API family is kept for source compatibility but is no longer the preferred runtime path: + +- `PatchCommandEncoder.apply(String, String)` +- `PatchCommandEncoder.apply(String, String, WordTraversalDirection)` +- `PatchCommandEncoder.applyTo(..., String, WordTraversalDirection, ...)` +- `PatchCommandEncoder.applyWithConfiguredDirection(String, String)` +- `StemmerPatchTrieLoader.load(...)` overloads returning `FrequencyTrie` +- `StemmerPatchTrieLoader.loadBinary(...)` overloads returning `FrequencyTrie` + +Use the compiled equivalents for runtime stemming: + +- `CompiledPatchCommand.compile(String, WordTraversalDirection)` +- `PatchCommandEncoder.compile(String)` +- `PatchCommandEncoder.compile(String, WordTraversalDirection)` +- `StemmerPatchTrieLoader.loadCompiled(...)` +- `StemmerPatchTrieLoader.loadBinaryCompiled(...)` + +## Loading A Text Dictionary + +Old runtime code: + +```java +Path dictionary = Path.of("dictionary.txt"); +ReductionSettings settings = ReductionSettings.withDefaults( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); +FrequencyTrie trie = StemmerPatchTrieLoader.load(dictionary, true, settings); + +String word = "running"; +String patch = trie.get(word); +String stem = patch == null + ? word + : PatchCommandEncoder.apply(word, patch, trie.traversalDirection()); +``` + +New runtime code: + +```java +Path dictionary = Path.of("dictionary.txt"); +ReductionSettings settings = ReductionSettings.withDefaults( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); +FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled(dictionary, true, settings); + +String word = "running"; +CompiledPatchCommand patch = trie.get(word); +String stem = patch == null ? word : patch.apply(word); +``` + +## Loading A Binary Artifact + +Old runtime code: + +```java +FrequencyTrie trie = StemmerPatchTrieLoader.loadBinary(Path.of("us-uk.radixor.gz")); + +String word = "studies"; +String patch = trie.get(word); +String stem = patch == null + ? word + : PatchCommandEncoder.apply(word, patch, trie.traversalDirection()); +``` + +New runtime code: + +```java +FrequencyTrie trie = + StemmerPatchTrieLoader.loadBinaryCompiled(Path.of("us-uk.radixor.gz")); + +String word = "studies"; +CompiledPatchCommand patch = trie.get(word); +String stem = patch == null ? word : patch.apply(word); +``` + +Existing binary artifacts remain readable. `loadBinaryCompiled(...)` reads the stored serialized patch strings and compiles them during load setup, before live stemming begins. + +## Manual Patch Encoding + +Encoding still produces a serialized patch command because that is the compact stored representation: + +```java +PatchCommandEncoder encoder = PatchCommandEncoder.builder().build(); +String patch = encoder.encode("running", "run"); +``` + +Old repeated application: + +```java +String stem = PatchCommandEncoder.apply("running", patch); +``` + +New repeated application: + +```java +CompiledPatchCommand compiled = encoder.compile(patch); +String stem = compiled.apply("running"); +``` + +## Caller-Owned Output Buffers + +Old buffer-oriented code: + +```java +char[] output = new char[32]; +int length = PatchCommandEncoder.applyTo( + "running", + patch, + WordTraversalDirection.BACKWARD, + output, + 0, + output.length); +``` + +New buffer-oriented code: + +```java +CompiledPatchCommand compiled = CompiledPatchCommand.compile(patch, WordTraversalDirection.BACKWARD); +char[] output = new char[32]; +int length = compiled.applyTo("running", output, 0, output.length); +``` + +Both APIs return `CompiledPatchCommand.APPLY_INSUFFICIENT_CAPACITY` when the caller-owned output range is too small. + +## Compatibility Rules + +Serialized patch strings remain part of the dictionary and artifact format. The deprecation is about repeated runtime application of serialized strings, not about the stored representation itself. + +Compatibility tests may continue to exercise the deprecated API to prove that old artifacts and source code still work during the transition. New production code, examples, and benchmark runtime paths should use `CompiledPatchCommand`. + +The command-line compiler still writes artifacts containing serialized patch commands. Runtime loaders can expose those commands as compiled immutable objects through `loadCompiled(...)` and `loadBinaryCompiled(...)`. + +## Contracted Trie Artifacts + +Current compiled loaders and freshly written binary artifacts can use contracted compiled tries. +Contraction replaces a subtree with an accepting leaf when every reachable entry below that subtree +selects the same preferred patch command. This changes the physical trie shape and the binary +stream version, but it does not change the serialized patch-command language. + +Existing binary artifacts remain readable through the compatibility reader. To obtain the +contracted runtime representation, rebuild the artifact with the current compiler or load the +source dictionary through the current `loadCompiled(...)` APIs. Applications that only consume +`CompiledPatchCommand` values through `get()` and `apply(...)` do not need code changes for this +optimization. diff --git a/docs/programmatic-extending-and-persistence.md b/docs/programmatic-extending-and-persistence.md index b568372..c4a6a1a 100644 --- a/docs/programmatic-extending-and-persistence.md +++ b/docs/programmatic-extending-and-persistence.md @@ -94,7 +94,8 @@ This model works especially well when domain-specific extensions are added in la After loading a compiled artifact, applications can inspect the persisted build descriptor directly: ```java -final FrequencyTrie trie = StemmerPatchTrieLoader.loadBinary("build/stemmers/cs_cz.dat.gz"); +final FrequencyTrie trie = + StemmerPatchTrieLoader.loadBinaryCompiled("build/stemmers/cs_cz.dat.gz"); final TrieMetadata metadata = trie.metadata(); System.out.println(metadata.formatVersion()); diff --git a/docs/programmatic-loading-and-building.md b/docs/programmatic-loading-and-building.md index 6269cda..9dbefae 100644 --- a/docs/programmatic-loading-and-building.md +++ b/docs/programmatic-loading-and-building.md @@ -4,11 +4,12 @@ This document explains how to acquire a compiled Radixor stemmer in Java. ## Load a bundled language dictionary -Bundled language resources are simple to use and compile directly into a `FrequencyTrie` during loading. +Bundled language resources are simple to use and compile directly into a `FrequencyTrie` during loading. ```java import java.io.IOException; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -20,7 +21,7 @@ public final class BundledLanguageExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( StemmerPatchTrieLoader.Language.US_UK, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); @@ -30,6 +31,11 @@ public final class BundledLanguageExample { The `storeOriginal` flag controls whether the canonical stem is inserted as a no-op patch entry for the stem itself. +Bundled `loadCompiled(...)` entry points build the runtime trie with the same contracted +representation used by the published benchmarks. During compilation, uniform preferred-command +subtrees are collapsed into accepting leaves, so lookup can stop before consuming the entire input +when the remaining characters cannot change the selected patch command. + ## Load a textual dictionary Loading from a dictionary file follows the same preparation model as bundled resources, but the source comes from your own file or path. The input may be plain UTF-8 text or GZip-compressed UTF-8 text; the loader detects GZip data from the stream header. The textual format is tab-separated values, meaning that columns are separated by the tab character. Each non-empty logical line starts with the stem column and may contain zero or more variant columns. Input case normalization is controlled by `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), trailing remarks introduced by `#` or `//` are ignored, and dictionary items containing embedded whitespace are currently ignored with warning-level diagnostics. @@ -38,6 +44,7 @@ Loading from a dictionary file follows the same preparation model as bundled res import java.io.IOException; import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.ReductionSettings; @@ -50,7 +57,7 @@ public final class LoadTextDictionaryExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( Path.of("data", "stemmer.tsv"), true, ReductionSettings.withDefaults( @@ -59,7 +66,12 @@ public final class LoadTextDictionaryExample { } ``` -Additional `StemmerPatchTrieLoader.load(...)` overloads let callers provide explicit `WordTraversalDirection`, `CaseProcessingMode`, `DiacriticProcessingMode`, or a complete `TrieMetadata` instance. Use those overloads when a custom dictionary must be compiled with forward traversal for right-to-left languages, case-sensitive keys, or diacritic stripping. +Additional `StemmerPatchTrieLoader.loadCompiled(...)` overloads let callers provide explicit `WordTraversalDirection`, `CaseProcessingMode`, `DiacriticProcessingMode`, or a complete `TrieMetadata` instance. Use those overloads when a custom dictionary must be compiled with forward traversal for right-to-left languages, case-sensitive keys, or diacritic stripping. + +When `ReductionSettings` are supplied through these compiled loader APIs, uniform-subtree +contraction is still enabled as an internal pre-reduction step. The public `ReductionMode` remains +the semantic policy for subtree equivalence after that contraction has removed regions whose +preferred command is already uniform. ## Load a compiled binary artifact @@ -69,6 +81,7 @@ Binary loading is typically the preferred runtime path because it avoids reparsi import java.io.IOException; import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -79,7 +92,7 @@ public final class LoadBinaryExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz")); } } @@ -96,6 +109,7 @@ at load time. This does not change persisted metadata, only the materialized in- import java.io.IOException; import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -106,14 +120,14 @@ public final class LoadBinaryWithDenseLookupExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie balanced = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie balanced = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz")); - final FrequencyTrie fast = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie fast = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 1024); - final FrequencyTrie compact = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie compact = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 0); } diff --git a/docs/programmatic-querying-and-ambiguity.md b/docs/programmatic-querying-and-ambiguity.md index 1309ae6..1b6d91a 100644 --- a/docs/programmatic-querying-and-ambiguity.md +++ b/docs/programmatic-querying-and-ambiguity.md @@ -10,7 +10,7 @@ This document explains how a compiled Radixor trie is queried and how ambiguity ```java final String word = "running"; -final String patch = trie.get(word); +final CompiledPatchCommand patch = trie.get(word); ``` ### `getAll(...)`: ordered local values @@ -18,7 +18,7 @@ final String patch = trie.get(word); `FrequencyTrie.getAll(String)` returns all values stored at the addressed node, ordered by descending frequency using the same deterministic tie-breaking rules. The returned array is a defensive copy. If the key is missing or has no local values, an empty array is returned. ```java -final String[] patches = trie.getAll("axes"); +final CompiledPatchCommand[] patches = trie.getAll("axes"); ``` ### `getEntries(...)`: values with counts @@ -28,9 +28,10 @@ final String[] patches = trie.getAll("axes"); ```java import java.util.List; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.ValueCount; -final List> entries = trie.getEntries("axes"); +final List> entries = trie.getEntries("axes"); ``` ### Visitor lookup for hot paths @@ -55,31 +56,29 @@ trie.getAllNormalized(token, 0, token.length, (patch, count, rank) -> { `getAllNormalized(...)` bypasses `caseProcessingMode` and `diacriticProcessingMode`; callers are responsible for supplying canonical input. `maxResults == 0` visits nothing, negative values are rejected, and a sink returning `false` stops iteration after the current callback. -## Apply patch commands +## Apply compiled patch commands -A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch. +A patch command is not the final stem. It must be applied to the original input token. Runtime code should use `CompiledPatchCommand`, which parses the stored patch-command representation once during setup and then applies the concrete immutable command repeatedly. ```java -import org.egothor.stemmer.PatchCommandEncoder; +import org.egothor.stemmer.CompiledPatchCommand; final String word = "running"; -final String patch = trie.get(word); -final String stem = PatchCommandEncoder.apply(word, patch); +final CompiledPatchCommand patch = trie.get(word); +final String stem = patch == null ? word : patch.apply(word); ``` Hot paths can apply a patch into caller-owned character storage: ```java final char[] output = new char[32]; -final int produced = PatchCommandEncoder.applyTo( +final int produced = patch.applyTo( word, - patch, - trie.traversalDirection(), output, 0, output.length); -if (produced != PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) { +if (produced != CompiledPatchCommand.APPLY_INSUFFICIENT_CAPACITY) { final String stem = new String(output, 0, produced); } ``` @@ -90,12 +89,14 @@ For multiple candidates: ```java final String word = "axes"; -for (final String patch : trie.getAll(word)) { - final String stem = PatchCommandEncoder.apply(word, patch); +for (final CompiledPatchCommand patch : trie.getAll(word)) { + final String stem = patch.apply(word); System.out.println(word + " -> " + stem + " (" + patch + ")"); } ``` +The historical `PatchCommandEncoder.apply(...)` API still exists for compatibility with code that directly handles serialized patch-command strings, but it is deprecated because it reparses the command on every call. See [Migration and Backward Compatibility](migration-and-backward-compatibility.md) for the old and new forms side by side. + ## Understand reduction modes Reduction mode determines how mutable subtrees are merged during compilation. All modes operate on full subtree semantics rather than only on local node content. diff --git a/docs/programmatic-usage.md b/docs/programmatic-usage.md index fdeaf44..fe109d8 100644 --- a/docs/programmatic-usage.md +++ b/docs/programmatic-usage.md @@ -24,6 +24,8 @@ This is why Radixor can generalize beyond explicitly listed forms and why compil The programmatic API is easier to understand when split by developer task: +- [Fast Track](fast-track.md) gives the shortest dependency-to-first-stem path for a new Java project. +- [Integration Deep Dive](integration-deep-dive.md) explains production integration, deployment artifacts, search-pipeline usage, and operational decisions. - [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage. - [Lookup Edge Optimization](lookup-edge-optimization.md) explains dense child lookup tuning and the speed/memory trade-off when materializing compiled tries. - [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes. @@ -35,7 +37,8 @@ The main types involved in programmatic usage are: - `FrequencyTrie.Builder` for mutable construction and extension, - `FrequencyTrie` for the compiled read-only trie, -- `PatchCommandEncoder` for creating and applying patch commands, +- `PatchCommandEncoder` for creating serialized patch commands, +- `CompiledPatchCommand` for repeated runtime patch application, - `StemmerPatchTrieLoader` for loading bundled or textual dictionaries, - `StemmerPatchTrieBinaryIO` for reading and writing compressed binary artifacts, - `FrequencyTrieBuilders` for reconstructing a mutable builder from a compiled trie, @@ -63,9 +66,11 @@ The core module is standalone and can be consumed directly as a normal Java modu For most developers, the best order is: -1. [Loading and Building Stemmers](programmatic-loading-and-building.md) -2. [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) -3. [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) +1. [Fast Track](fast-track.md) +2. [Integration Deep Dive](integration-deep-dive.md) +3. [Loading and Building Stemmers](programmatic-loading-and-building.md) +4. [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) +5. [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) ## Next steps diff --git a/docs/quick-start.md b/docs/quick-start.md index 71010d2..294409a 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -2,6 +2,10 @@ This guide introduces the fastest practical path to using **Radixor**. +If you are new to Radixor and want the shortest possible path to a first working stem, start with +[Fast Track](fast-track.md). This Quick Start is a broader developer walkthrough: it introduces the +main loading options, query methods, artifact workflow, and metadata model. + Radixor separates preparation from runtime usage. Source dictionaries are used to derive patch commands and reduce them into a compact read-only trie. Runtime stemming then operates on that compiled structure rather than on the original dictionary text. A richer dictionary usually improves the quality and coverage of inferred transformations, including transformations that are applicable to words not explicitly present in the source material. The reduction step also removes a large amount of redundant lexical information, which is why very large dictionaries can still produce compact runtime artifacts. These artifacts can be persisted and loaded directly when needed. A practical workflow usually consists of two independent phases: @@ -15,11 +19,12 @@ A compiled stemmer can be obtained in three common ways. ### Use a bundled language dictionary -Radixor ships with bundled dictionaries for a set of supported languages. These resources are line-oriented dictionaries stored with the library and compiled into a `FrequencyTrie` when loaded. The loader can also store the canonical stem itself as a no-op patch command. Compiled trie artifacts now persist self-describing metadata, including the traversal direction and compilation reduction settings used to build the artifact. +Radixor ships with bundled dictionaries for a set of supported languages. These resources are line-oriented dictionaries stored with the library and compiled into a `FrequencyTrie` when loaded through the runtime API. The loader can also store the canonical stem itself as a no-op patch command. Compiled trie artifacts now persist self-describing metadata, including the traversal direction and compilation reduction settings used to build the artifact. ```java import java.io.IOException; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -31,7 +36,7 @@ public final class BundledStemmerExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( StemmerPatchTrieLoader.Language.US_UK, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); @@ -49,6 +54,7 @@ Compiled stemmers can be stored as GZip-compressed binary artifacts and loaded d import java.io.IOException; import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -59,7 +65,7 @@ public final class LoadBinaryStemmerExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz")); System.out.println("Canonical node count: " + trie.size()); @@ -73,6 +79,7 @@ You can tune in-memory child lookup density at load time without changing the ar import java.io.IOException; import java.nio.file.Path; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -83,10 +90,10 @@ public final class LoadBinaryStemmerExampleTuned { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie fast = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie fast = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 1024); - final FrequencyTrie compact = StemmerPatchTrieLoader.loadBinary( + final FrequencyTrie compact = StemmerPatchTrieLoader.loadBinaryCompiled( Path.of("stemmers", "english.radixor.gz"), 128); @@ -107,7 +114,7 @@ A dedicated CLI compilation workflow deserves its own focused page and should re ## 2. Use the compiled stemmer -A compiled `FrequencyTrie` stores patch commands, not final stems. Querying therefore has two steps: +A compiled `FrequencyTrie` stores patch commands, not final stems. Querying therefore has two steps: 1. retrieve one or more patch commands from the trie, 2. apply each patch command to the original input word. @@ -121,8 +128,8 @@ Use `get(...)` when the application needs a single preferred transformation. ```java import java.io.IOException; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.FrequencyTrie; -import org.egothor.stemmer.PatchCommandEncoder; import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.StemmerPatchTrieLoader; @@ -133,14 +140,14 @@ public final class SingleStemExample { } public static void main(final String[] arguments) throws IOException { - final FrequencyTrie trie = StemmerPatchTrieLoader.load( + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled( StemmerPatchTrieLoader.Language.US_UK, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); final String word = "running"; - final String patch = trie.get(word); - final String stem = PatchCommandEncoder.apply(word, patch); + final CompiledPatchCommand patch = trie.get(word); + final String stem = patch == null ? word : patch.apply(word); System.out.println(word + " -> " + stem + " (" + patch + ")"); } @@ -153,10 +160,10 @@ Use `getAll(...)` when the application should preserve ambiguity instead of coll ```java final String word = "axes"; -final String[] patches = trie.getAll(word); +final CompiledPatchCommand[] patches = trie.getAll(word); -for (final String patch : patches) { - final String stem = PatchCommandEncoder.apply(word, patch); +for (final CompiledPatchCommand patch : patches) { + final String stem = patch.apply(word); System.out.println(word + " -> " + stem + " (" + patch + ")"); } ``` @@ -168,11 +175,12 @@ For diagnostics or advanced ranking logic, use `getEntries(...)` to obtain value ```java import java.util.List; +import org.egothor.stemmer.CompiledPatchCommand; import org.egothor.stemmer.ValueCount; -final List> entries = trie.getEntries("axes"); +final List> entries = trie.getEntries("axes"); -for (final ValueCount entry : entries) { +for (final ValueCount entry : entries) { System.out.println(entry.value() + " -> " + entry.count()); } ``` @@ -210,7 +218,11 @@ public final class ExtendCompiledStemmerExample { String[]::new, settings); - builder.put("microservices", "Na"); + final PatchCommandEncoder encoder = PatchCommandEncoder.builder() + .traversalDirection(compiledTrie.traversalDirection()) + .build(); + + builder.put("microservices", encoder.encode("microservices", "microservice")); final FrequencyTrie updatedTrie = builder.build(); diff --git a/docs/reduction-semantics.md b/docs/reduction-semantics.md index b81b0b9..1c435c0 100644 --- a/docs/reduction-semantics.md +++ b/docs/reduction-semantics.md @@ -103,6 +103,24 @@ This turns reduction into a canonicalization process: That is how Radixor eliminates duplicated equivalent subtrees. +## Uniform-subtree contraction + +Radixor performs one additional internal reduction before each public reduction mode is applied. +When all reachable entries below a subtree have the same preferred patch command, the subtree is +contracted into an accepting leaf for that command. + +This optimization is deliberately narrower than the public reduction modes: + +- it is based on preferred `get()` behavior, +- it does not depend on child edge shape once the preferred command is uniform, +- it removes lookup depth that cannot affect the selected command, +- it preserves the standard single-result stemming path used by `StemmerPatchTrieLoader.loadCompiled(...)`. + +The effect is especially visible in large dictionary tries with many inflected forms that map to +the same command class, such as no-op preservation or common suffix deletion. Runtime lookup can +return the accepting leaf as soon as it is reached instead of traversing the remaining characters +only to discover the same command deeper in the trie. + ## Count aggregation and compiled state When multiple original build-time subtrees collapse into one canonical reduced node, local counts may be aggregated. diff --git a/docs/reports.md b/docs/reports.md index bb0a467..3516178 100644 --- a/docs/reports.md +++ b/docs/reports.md @@ -42,15 +42,14 @@ They are useful for dependency inspection, downstream integration, compliance-or ## Benchmark outputs and badge metadata -These resources expose benchmark results and generated badge metadata derived from the latest published build: +These resources expose benchmark results and generated badge metadata derived from the latest published build. JMH benchmark reports are published as TXT and CSV files; the historical Porter comparison badge is no longer generated. - [JMH benchmark results (TXT)](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt) - [JMH benchmark results (CSV)](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.csv) - [Coverage badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json) - [Mutation badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json) -- [Benchmark badge metadata](https://leogalambos.github.io/Radixor/builds/latest/metrics/jmh-badge.json) -The benchmark outputs provide direct access to the published JMH result files, while the badge metadata endpoints are intended for status surfaces such as the project README or other generated dashboards. +The benchmark outputs provide direct access to the published JMH result files. Coverage and mutation badge metadata endpoints are intended for status surfaces such as the project README or other generated dashboards. ## Practical usage diff --git a/docs/test-taxonomy-and-filtering.md b/docs/test-taxonomy-and-filtering.md index 8668de4..e6241bc 100644 --- a/docs/test-taxonomy-and-filtering.md +++ b/docs/test-taxonomy-and-filtering.md @@ -145,6 +145,14 @@ defensive fallback in case of future tag drift. All examples use Gradle with JUnit Platform integration: +- Default fast test run: + +``` +./gradlew test +``` + +The default `test` task excludes `slow` tests. Supplying `-DincludeTags` or `-PincludeTags` still excludes `slow` unless the include expression contains `slow`; supplying an explicit exclude expression replaces the default. Long-running bundled-dictionary compilation and full-language loading checks therefore run only through an explicit tag expression such as `-DincludeTags=slow` or a dedicated profile such as `ciSlow`. + - Only unit tests: ``` @@ -154,7 +162,7 @@ All examples use Gradle with JUnit Platform integration: - Integration tests only: ``` -./gradlew test -DincludeTags=integration +./gradlew test -DincludeTags=integration -DexcludeTags=slow ``` - Only trie subsystem tests: diff --git a/gradle.lockfile b/gradle.lockfile index 0b70d2e..1e65d4f 100644 --- a/gradle.lockfile +++ b/gradle.lockfile @@ -24,7 +24,15 @@ org.apache.commons:commons-lang3:3.18.0=pitest org.apache.commons:commons-lang3:3.20.0=pmd org.apache.commons:commons-math3:3.6.1=jmh,jmhCompileClasspath,jmhRuntimeClasspath org.apache.commons:commons-text:1.14.0=pitest +org.apache.lucene:lucene-analysis-common:10.5.0=jmhCompileClasspath,jmhRuntimeClasspath +org.apache.lucene:lucene-analysis-morfologik:10.5.0=jmhCompileClasspath,jmhRuntimeClasspath +org.apache.lucene:lucene-analysis-stempel:10.5.0=jmhCompileClasspath,jmhRuntimeClasspath +org.apache.lucene:lucene-core:10.5.0=jmhCompileClasspath,jmhRuntimeClasspath +org.apache.opennlp:opennlp-tools:2.5.4=jmhCompileClasspath,jmhRuntimeClasspath org.apiguardian:apiguardian-api:1.1.2=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.carrot2:morfologik-fsa:2.1.9=jmhCompileClasspath,jmhRuntimeClasspath +org.carrot2:morfologik-polish:2.1.9=jmhRuntimeClasspath +org.carrot2:morfologik-stemming:2.1.9=jmhCompileClasspath,jmhRuntimeClasspath org.checkerframework:checker-qual:3.52.1=pmd org.jacoco:org.jacoco.agent:0.8.14=jacocoAgent,jacocoAnt org.jacoco:org.jacoco.ant:0.8.14=jacocoAnt @@ -62,5 +70,7 @@ org.pitest:pitest-html-report:1.22.1=pitest org.pitest:pitest-junit5-plugin:1.2.3=pitest org.pitest:pitest:1.22.1=pitest org.slf4j:jul-to-slf4j:1.7.36=pmd +org.slf4j:slf4j-api:2.0.17=jmhCompileClasspath,jmhRuntimeClasspath org.xmlresolver:xmlresolver:5.3.3=pmd +ua.net.nlp:morfologik-ukrainian-search:4.9.1=jmhRuntimeClasspath empty=annotationProcessor,compileClasspath,cyclonedxBom,jmhAnnotationProcessor,mainPmdAuxClasspath,runtimeClasspath,testAnnotationProcessor diff --git a/gradle/lucene-benchmarks.gradle b/gradle/lucene-benchmarks.gradle new file mode 100644 index 0000000..f9eeb10 --- /dev/null +++ b/gradle/lucene-benchmarks.gradle @@ -0,0 +1,223 @@ +import org.gradle.plugins.ide.eclipse.model.SourceFolder + + +def luceneVersion = '10.5.0' +def luceneRootRelativePath = 'third-party/lucene' +def luceneSourceArtifacts = ['lucene-analysis-common', 'lucene-analyzers-common'] +def luceneSourceDirectory = layout.buildDirectory.dir("${luceneRootRelativePath}/source/analyzers-common") + +def luceneGeneratedSourceDirectory = layout.buildDirectory.dir('generated/sources/lucene') +def luceneGeneratedPorterFile = luceneGeneratedSourceDirectory.map { it.file('org/egothor/stemmer/benchmark/LucenePorterStemmerCopied.java') } + +def luceneSourceDownloadFile = layout.buildDirectory.file("${luceneRootRelativePath}/lucene-${luceneVersion}-sources.jar") + +dependencies { + jmhImplementation "org.apache.lucene:lucene-analysis-common:${luceneVersion}" + jmhImplementation "org.apache.lucene:lucene-analysis-stempel:${luceneVersion}" + jmhImplementation "org.apache.lucene:lucene-analysis-morfologik:${luceneVersion}" +} + +def buildLuceneSourcesName = { final String artifact -> + "${artifact}-${luceneVersion}-sources.jar" +} +def buildLuceneSourcesUrl = { final String artifact -> + "https://repo1.maven.org/maven2/org/apache/lucene/${artifact}/${luceneVersion}/${buildLuceneSourcesName(artifact)}" +} + +def isLuceneSourcesDownloadable = { final String artifact -> + try { + final URL sourceUrl = new URL(buildLuceneSourcesUrl(artifact)) + final java.net.HttpURLConnection connection = (java.net.HttpURLConnection) sourceUrl.openConnection() + connection.requestMethod = 'HEAD' + connection.instanceFollowRedirects = true + connection.connectTimeout = 10000 + connection.readTimeout = 10000 + final int responseCode = connection.responseCode + connection.disconnect() + return responseCode == 200 + } catch (Exception ignored) { + return false + } +} + +def downloadLuceneSourcesJar = { -> + final File targetFile = luceneSourceDownloadFile.get().asFile + + for (String artifact : luceneSourceArtifacts) { + if (!isLuceneSourcesDownloadable(artifact)) { + continue + } + + final String sourceUrl = buildLuceneSourcesUrl(artifact) + final File tempFile = new File(targetFile.parentFile, "${artifact}.${luceneVersion}.tmp") + try { + new URL(sourceUrl).withInputStream { inputStream -> + tempFile.parentFile.mkdirs() + tempFile.withOutputStream { outputStream -> + outputStream << inputStream + } + } + if (!tempFile.exists() || tempFile.length() <= 0L) { + throw new GradleException("Downloaded Lucene source artifact for ${artifact} was empty.") + } + targetFile.delete() + if (!tempFile.renameTo(targetFile)) { + throw new GradleException("Failed to persist downloaded Lucene source artifact for ${artifact}.") + } + return + } catch (Exception ignored) { + tempFile.delete() + } + } + + throw new GradleException( + "Failed to download Apache Lucene source artifacts ${luceneSourceArtifacts} for version ${luceneVersion}.") +} + +def luceneSourceClasspathPath = provider { + project.relativePath(luceneGeneratedSourceDirectory.get().asFile) +} +def luceneEclipseClasspathAttributes = [ + gradle_scope : 'jmh', + gradle_used_by_scope: 'jmh', + test : 'true' +] + +def isAbsoluteClasspathPath = { String path -> + path.startsWith('/') || path ==~ /^[A-Za-z]:[\\\/].*/ +} + +def luceneGeneratedPorterNotice = ''' +/** + * Generated at benchmark execution time from Apache Lucene source. + * + * This source copy is compiled only for the JMH benchmark source set and is + * not committed as production code. + */ +''' + +def transformPorterStemmerSource = { final File sourceFile, final File targetFile -> + if (!sourceFile.exists()) { + throw new GradleException("Apache Lucene PorterStemmer source was not available at ${sourceFile}.") + } + + final String sourceText = sourceFile.getText('UTF-8') + String transformedText = sourceText + + if (transformedText.contains('package org.apache.lucene.analysis.en;')) { + transformedText = transformedText.replaceFirst(/(?m)^\s*package\s+org\.apache\.lucene\.analysis\.en\s*;/, + 'package org.egothor.stemmer.benchmark;') + } else { + throw new GradleException( + 'Expected Lucene package-private PorterStemmer in org.apache.lucene.analysis.en package was not found in downloaded source.') + } + + transformedText = transformedText.replaceFirst(/(?m)^\s*class\s+PorterStemmer\s*\{/, 'public final class LucenePorterStemmerCopied {') + transformedText = transformedText.replaceFirst(/(?m)^\s*public\s+PorterStemmer\(\)/, 'public LucenePorterStemmerCopied()') + + if (!transformedText.contains('class LucenePorterStemmerCopied')) { + throw new GradleException("Failed to rename PorterStemmer class when generating ${targetFile}.") + } + + targetFile.parentFile.mkdirs() + targetFile.text = transformedText +} + +def resolveLucenePorterStemmerSource = { -> + final File sourceRoot = luceneSourceDirectory.get().asFile + final List candidates = [ + 'org/apache/lucene/analysis/en/org/apache/lucene/analysis/en/PorterStemmer.java', + 'org/apache/lucene/analysis/en/PorterStemmer.java', + 'org/apache/lucene/analysis/en/org/tartarus/snowball/ext/PorterStemmer.java' + ] + + for (String candidate : candidates) { + final File file = new File(sourceRoot, candidate) + if (file.exists()) { + return file + } + } + + final FileTree porterCandidates = fileTree(sourceRoot).matching { include '**/PorterStemmer.java' } + for (File file : porterCandidates.files) { + if (file.text.contains('class PorterStemmer') && file.text.contains('package org.apache.lucene.analysis.en;')) { + return file + } + } + + throw new GradleException('Unable to resolve Lucene PorterStemmer source file from extracted artifact.') +} + +tasks.register('downloadLuceneAnalyzersSources') { + group = 'build setup' + description = 'Downloads Apache Lucene analysis sources for benchmark-only code generation.' + + outputs.file(luceneSourceDownloadFile) + + doLast { + if (!luceneSourceDownloadFile.get().asFile.exists()) { + downloadLuceneSourcesJar() + } + } +} + +tasks.register('extractLuceneAnalyzersSources', Copy) { + group = 'build setup' + description = 'Extracts Apache Lucene analysis source JAR for benchmark-only extraction.' + + dependsOn(tasks.named('downloadLuceneAnalyzersSources')) + + from(zipTree(luceneSourceDownloadFile)) + into(luceneSourceDirectory) +} + +tasks.register('generateLucenePorterStemmerCopied') { + group = 'build setup' + description = 'Generates LucenePorterStemmerCopied into the build-only benchmark source directory.' + + dependsOn(tasks.named('extractLuceneAnalyzersSources')) + + inputs.dir(luceneSourceDirectory) + outputs.file(luceneGeneratedPorterFile) + + doLast { + final File sourceFile = resolveLucenePorterStemmerSource() + transformPorterStemmerSource(sourceFile, luceneGeneratedPorterFile.get().asFile) + } +} + +sourceSets { + jmh { + java { + srcDir(luceneGeneratedSourceDirectory) + } + } +} + +tasks.named('compileJmhJava') { + dependsOn(tasks.named('generateLucenePorterStemmerCopied')) +} + +eclipse { + classpath { + file { + whenMerged { classpath -> + String generatedPath = luceneSourceClasspathPath.get() + + classpath.entries.removeAll { entry -> + entry.hasProperty('path') && ( + entry.path == generatedPath || + isAbsoluteClasspathPath(entry.path) + ) + } + + SourceFolder luceneEntry = new SourceFolder(generatedPath, null) + luceneEntry.output = 'bin/jmh' + luceneEclipseClasspathAttributes.each { String name, String value -> + luceneEntry.entryAttributes[name] = value + } + classpath.entries.add(luceneEntry) + } + } + } +} diff --git a/gradle/opennlp-benchmarks.gradle b/gradle/opennlp-benchmarks.gradle new file mode 100644 index 0000000..cdcfde8 --- /dev/null +++ b/gradle/opennlp-benchmarks.gradle @@ -0,0 +1,5 @@ +def openNlpVersion = '2.5.4' + +dependencies { + jmhImplementation "org.apache.opennlp:opennlp-tools:${openNlpVersion}" +} diff --git a/gradle/paicehusk-benchmarks.gradle b/gradle/paicehusk-benchmarks.gradle new file mode 100644 index 0000000..b21c903 --- /dev/null +++ b/gradle/paicehusk-benchmarks.gradle @@ -0,0 +1,219 @@ +import org.gradle.plugins.ide.eclipse.model.SourceFolder + + +def paicehuskVersion = 'master' +def paicehuskArchiveName = "paice-husk-stemmer-${paicehuskVersion}.zip" +def paicehuskDownloadUrl = "https://github.com/Hopper262/paice-husk-stemmer/archive/refs/heads/${paicehuskVersion}.zip" +def paicehuskDownloadFile = layout.buildDirectory.file("third-party/paicehusk/${paicehuskArchiveName}") +def paicehuskExtractDirectory = layout.buildDirectory.dir('third-party/paicehusk/source') +def paicehuskArchiveDirectory = paicehuskExtractDirectory.map { it.dir('paice-husk-stemmer-master') } +def paicehuskJavaFile = paicehuskArchiveDirectory.map { it.file('paicehusk_java.java') } +def paicehuskRulesFile = paicehuskArchiveDirectory.map { it.file('paicehusk_rules.txt') } + +def paicehuskGeneratedSourceDirectory = layout.buildDirectory.dir('generated/sources/paicehusk') +def paicehuskGeneratedStemmerFile = paicehuskGeneratedSourceDirectory.map { it.file('org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmer.java') } + +def paicehuskGeneratedSourcePath = provider { + project.relativePath(paicehuskGeneratedSourceDirectory.get().asFile) +} +def paicehuskSourceEclipseClasspathAttributes = [ + gradle_scope : 'jmh', + gradle_used_by_scope: 'jmh', + test : 'true' +] + +def paicehuskIsAbsolutePath = { String path -> + path.startsWith('/') || path ==~ /^[A-Za-z]:[\\\/].*/ +} + +def paicehuskGeneratedNotice = ''' +/** + * Generated at benchmark execution time from upstream + * https://github.com/Hopper262/paice-husk-stemmer . + * + * This source copy is compiled only for the JMH benchmark source set and is + * not committed as production code. + */ +''' + +def escapeForJava = { final String text -> + return text.replace('\\\\', '\\\\\\\\') + .replace('\"', '\\\"') +} + +def toRuleLines = { final File rulesFile -> + final List lines = rulesFile.readLines('UTF-8') + final StringBuilder ruleLines = new StringBuilder() + for (int index = 0; index < lines.size(); index++) { + final String line = lines.get(index) + ruleLines.append(' "') + ruleLines.append(escapeForJava(line)) + ruleLines.append('"') + if (index < lines.size() - 1) { + ruleLines.append(',') + } + ruleLines.append('\n') + } + return ruleLines.toString() +} + +def paicehuskEngineInsertion = { final String ruleLines -> + return """ + + public static final String[] RULE_LINES = { +${ruleLines} + }; + + private static final java.util.HashMap RULES = createRulesFromEmbeddedRules(); + + /** + * Creates benchmark stemmer instance. + */ + public PaiceHuskLancasterStemmer() { + } + + /** + * Applies Paice/Husk stemming to one token. + * + * @param token input token + * @return stemmed token + */ + public String stem(final String token) { + if (token == null) { + return null; + } + return stemWord(token, RULES, null); + } + + /** + * Loads bundled rule lines directly from the generated benchmark source. + * + * @return initialized rule map + */ + private static java.util.HashMap createRulesFromEmbeddedRules() { + try { + final java.io.File ruleFile = java.io.File.createTempFile("paicehusk-rules", ".txt"); + ruleFile.deleteOnExit(); + try (java.io.PrintWriter writer = new java.io.PrintWriter(new java.io.FileWriter(ruleFile))) { + for (String line : RULE_LINES) { + writer.println(line); + } + } + return loadRules(ruleFile.getAbsolutePath()); + } catch (Exception exception) { + throw new IllegalStateException("Unable to initialize benchmark Paice/Husk rules.", exception); + } + } +""" +} + +def transformPaiceHuskSource = { final File sourceFile, final File rulesFile, final File targetFile -> + if (!sourceFile.exists()) { + throw new GradleException("Paice/Husk Java source was not available at ${sourceFile}.") + } + if (!rulesFile.exists()) { + throw new GradleException("Paice/Husk rule file was not available at ${rulesFile}.") + } + + final String sourceText = sourceFile.getText('UTF-8') + String transformedText = sourceText + + transformedText = 'package org.egothor.stemmer.benchmark;' + '\n\n' + transformedText + transformedText = transformedText.replaceFirst(/(?m)^\s*class\s+PaiceHusk\s*\{/, 'public final class PaiceHuskLancasterStemmer {') + + final int packageEnd = transformedText.indexOf('\n', transformedText.indexOf('package org.egothor.stemmer.benchmark;')) + if (packageEnd >= 0) { + transformedText = transformedText.substring(0, packageEnd + 1) + '\n' + paicehuskGeneratedNotice + transformedText.substring(packageEnd + 1) + } + + final String marker = '\n} // end class PaiceHusk' + final int markerIndex = transformedText.lastIndexOf(marker) + if (markerIndex < 0) { + throw new GradleException("Unexpected Paice/Husk source structure at ${sourceFile}.") + } + final String replacement = paicehuskEngineInsertion(toRuleLines(rulesFile)) + transformedText = transformedText.substring(0, markerIndex) + '\n' + replacement + '\n}' + + targetFile.parentFile.mkdirs() + targetFile.text = transformedText +} + +tasks.register('downloadPaiceHuskStemmer') { + group = 'build setup' + description = 'Downloads the upstream Paice/Husk benchmark source for dynamic extraction.' + + outputs.file(paicehuskDownloadFile) + + doLast { + final File targetFile = paicehuskDownloadFile.get().asFile + targetFile.parentFile.mkdirs() + + if (!targetFile.exists()) { + new URL(paicehuskDownloadUrl).withInputStream { inputStream -> + targetFile.withOutputStream { outputStream -> + outputStream << inputStream + } + } + } + } +} + +tasks.register('extractPaiceHuskStemmer', Copy) { + group = 'build setup' + description = 'Extracts the upstream Paice/Husk benchmark archive.' + + dependsOn(tasks.named('downloadPaiceHuskStemmer')) + + from(zipTree(paicehuskDownloadFile)) + into(paicehuskExtractDirectory) +} + +tasks.register('generatePaiceHuskLancasterStemmer') { + group = 'build setup' + description = 'Generates PaiceHuskLancasterStemmer into a benchmark-only generated source directory.' + + dependsOn(tasks.named('extractPaiceHuskStemmer')) + + inputs.files(paicehuskJavaFile, paicehuskRulesFile) + outputs.file(paicehuskGeneratedStemmerFile) + + doLast { + transformPaiceHuskSource(paicehuskJavaFile.get().asFile, paicehuskRulesFile.get().asFile, paicehuskGeneratedStemmerFile.get().asFile) + } +} + +sourceSets { + jmh { + java { + srcDir(paicehuskGeneratedSourceDirectory) + } + } +} + +tasks.named('compileJmhJava') { + dependsOn(tasks.named('generatePaiceHuskLancasterStemmer')) +} + +eclipse { + classpath { + file { + whenMerged { classpath -> + String generatedPath = paicehuskGeneratedSourcePath.get() + + classpath.entries.removeAll { entry -> + entry.hasProperty('path') && ( + entry.path == generatedPath || + paicehuskIsAbsolutePath(entry.path) + ) + } + + SourceFolder paicehuskEntry = new SourceFolder(generatedPath, null) + paicehuskEntry.output = 'bin/jmh' + paicehuskSourceEclipseClasspathAttributes.each { String name, String value -> + paicehuskEntry.entryAttributes[name] = value + } + classpath.entries.add(paicehuskEntry) + } + } + } +} diff --git a/gradle/snowball-benchmarks.gradle b/gradle/snowball-benchmarks.gradle index c1b0231..c99b510 100644 --- a/gradle/snowball-benchmarks.gradle +++ b/gradle/snowball-benchmarks.gradle @@ -7,13 +7,43 @@ def snowballDistributionDirectoryName = "libstemmer_java-${snowballVersion}" def snowballRootRelativePath = 'third-party/snowball' def snowballSourceRelativePath = "${snowballRootRelativePath}/source" def snowballJavaSourceRelativePath = "${snowballSourceRelativePath}/${snowballDistributionDirectoryName}/java" +def snowballGeneratedSourceRelativePath = 'generated/sources/snowball' def snowballDownloadUrl = "https://snowballstem.org/dist/${snowballArchiveName}" def snowballDownloadFile = layout.buildDirectory.file("${snowballRootRelativePath}/${snowballArchiveName}") def snowballExtractDirectory = layout.buildDirectory.dir(snowballSourceRelativePath) def snowballJavaSourceDirectory = layout.buildDirectory.dir(snowballJavaSourceRelativePath) +def snowballGeneratedSourceDirectory = layout.buildDirectory.dir(snowballGeneratedSourceRelativePath) def snowballJavaSourceClasspathPath = provider { project.relativePath(snowballJavaSourceDirectory.get().asFile) } +def snowballGeneratedSourceClasspathPath = provider { + project.relativePath(snowballGeneratedSourceDirectory.get().asFile) +} +def transformSnowballSourceText = { final String sourceText -> + String transformedText = sourceText + + transformedText = transformedText.replaceAll(/(?m)^\s*package\s+org\.tartarus\.snowball\.ext\s*;/, + 'package org.egothor.stemmer.benchmark.snowball.ext;') + transformedText = transformedText.replaceAll(/(?m)^\s*package\s+org\.tartarus\.snowball\s*;/, + 'package org.egothor.stemmer.benchmark.snowball;') + transformedText = transformedText.replace('org.tartarus.snowball.', 'org.egothor.stemmer.benchmark.snowball.') + + return transformedText +} +def copySnowballSourcesWithPackageIsolation = { final File sourceDirectory, final File targetDirectory -> + final FileTree sourceFiles = fileTree(sourceDirectory).matching { include '**/*.java' } + if (targetDirectory.exists()) { + targetDirectory.deleteDir() + } + + for (File sourceFile : sourceFiles.files) { + final String relativePath = sourceDirectory.toPath().relativize(sourceFile.toPath()).toString() + final File outputFile = new File(targetDirectory, relativePath) + outputFile.parentFile.mkdirs() + outputFile.text = transformSnowballSourceText(sourceFile.getText('UTF-8')) + } +} + def snowballEclipseClasspathAttributes = [ gradle_scope : 'jmh', gradle_used_by_scope: 'jmh', @@ -53,16 +83,33 @@ tasks.register('extractSnowballJava', Copy) { into(snowballExtractDirectory) } +tasks.register('generateIsolatedSnowballSources') { + group = 'build setup' + description = 'Copies Snowball source to benchmark-only package-isolated package paths.' + + dependsOn(tasks.named('extractSnowballJava')) + + inputs.dir(snowballJavaSourceDirectory) + outputs.dir(snowballGeneratedSourceDirectory) + + doLast { + copySnowballSourcesWithPackageIsolation( + snowballJavaSourceDirectory.get().asFile, + snowballGeneratedSourceDirectory.get().asFile + ) + } +} + sourceSets { jmh { java { - srcDir(snowballJavaSourceDirectory) + srcDir(snowballGeneratedSourceDirectory) } } } tasks.named('compileJmhJava') { - dependsOn(tasks.named('extractSnowballJava')) + dependsOn(tasks.named('generateIsolatedSnowballSources')) } eclipse { @@ -70,17 +117,19 @@ eclipse { file { whenMerged { classpath -> String generatedSnowballPath = snowballJavaSourceClasspathPath.get() + String generatedIsolatedSnowballPath = snowballGeneratedSourceClasspathPath.get() String modelSnowballPath = snowballJavaSourceRelativePath classpath.entries.removeAll { entry -> entry.hasProperty('path') && ( entry.path == generatedSnowballPath || + entry.path == generatedIsolatedSnowballPath || entry.path == modelSnowballPath || isAbsoluteClasspathPath(entry.path) ) } - SourceFolder snowballEntry = new SourceFolder(generatedSnowballPath, null) + SourceFolder snowballEntry = new SourceFolder(generatedIsolatedSnowballPath, null) snowballEntry.output = 'bin/jmh' snowballEclipseClasspathAttributes.each { String name, String value -> snowballEntry.entryAttributes[name] = value diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index afb62e7..a00863b 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -61,16 +61,6 @@ - - - - - - - - - - @@ -136,11 +126,6 @@ - - - - - @@ -157,16 +142,6 @@ - - - - - - - - - - @@ -284,16 +259,6 @@ - - - - - - - - - - @@ -456,9 +421,6 @@ - - - @@ -710,6 +672,11 @@ + + + + + @@ -846,9 +813,6 @@ - - - @@ -905,6 +869,14 @@ + + + + + + + + @@ -913,6 +885,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -1106,6 +1102,19 @@ + + + + + + + + + + + + + @@ -1137,6 +1146,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1384,89 +1422,56 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1480,17 +1485,11 @@ - - - - - - @@ -1630,11 +1629,6 @@ - - - - - @@ -1955,6 +1949,14 @@ + + + + + + + + diff --git a/mkdocs.yml b/mkdocs.yml index a69c699..9268fdf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,17 +38,23 @@ markdown_extensions: nav: - Home: index.md - - Getting Started: + - Start: + - Fast Track: fast-track.md - Quick Start: quick-start.md - - Built-in Languages: built-in-languages.md - - Dictionary Format: dictionary-format.md - - CLI Compilation: cli-compilation.md + - Integration Deep Dive: integration-deep-dive.md - - Programmatic Usage: + - Integration: - Overview: programmatic-usage.md - Loading and Building Stemmers: programmatic-loading-and-building.md - Querying and Ambiguity Handling: programmatic-querying-and-ambiguity.md - Extending and Persisting Compiled Tries: programmatic-extending-and-persistence.md + - Migration and Backward Compatibility: migration-and-backward-compatibility.md + - CLI Compilation: cli-compilation.md + + - Dictionaries and Languages: + - Built-in Languages: built-in-languages.md + - Dictionary Format: dictionary-format.md + - Contributing Dictionaries: contributing-dictionaries.md - Architecture and Semantics: - Overview: architecture-and-reduction.md @@ -57,11 +63,38 @@ nav: - Lookup Edge Optimization: lookup-edge-optimization.md - Compatibility and Guarantees: compatibility-and-guarantees.md - - Dictionaries: - - Contributing Dictionaries: contributing-dictionaries.md + - Benchmarks: + - How to Read Benchmarks: benchmarking.md + - Benchmark Results: benchmarks/index.md + - Reference: + - Methodology: benchmarks/reference/methodology.md + - Corpora: benchmarks/reference/corpora.md + - Environment and Reports: benchmarks/reference/environment.md + - English Dictionary Coverage: benchmarks/reference/english-coverage.md + - Candidate Evaluation: benchmarks/reference/candidates.md + - Language Results: + - Overview: benchmarks/languages/index.md + - Czech: benchmarks/languages/czech.md + - Danish: benchmarks/languages/danish.md + - Dutch: benchmarks/languages/dutch.md + - English: benchmarks/languages/english.md + - Finnish: benchmarks/languages/finnish.md + - French: benchmarks/languages/french.md + - German: benchmarks/languages/german.md + - Hungarian: benchmarks/languages/hungarian.md + - Italian: benchmarks/languages/italian.md + - Norwegian Bokmal: benchmarks/languages/norwegian-bokmal.md + - Norwegian Nynorsk: benchmarks/languages/norwegian-nynorsk.md + - Persian: benchmarks/languages/persian.md + - Polish: benchmarks/languages/polish.md + - Portuguese: benchmarks/languages/portuguese.md + - Russian: benchmarks/languages/russian.md + - Spanish: benchmarks/languages/spanish.md + - Swedish: benchmarks/languages/swedish.md + - Ukrainian: benchmarks/languages/ukrainian.md + - Yiddish: benchmarks/languages/yiddish.md - Quality and Operations: - Quality and Operations: quality-and-operations.md - - Benchmarking: benchmarking.md - Reports: reports.md - Test taxonomy and execution filtering: test-taxonomy-and-filtering.md diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java index 583c020..1700b7d 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java @@ -113,12 +113,12 @@ final class BenchmarkCorpusSupport { dictionaryBuilder.append(stem); lookupKeys.add(stem); for (String variant : variants) { - dictionaryBuilder.append(' ').append(variant); + dictionaryBuilder.append('\t').append(variant); lookupKeys.add(variant); } final String homograph = createHomograph(index); - dictionaryBuilder.append(' ').append(homograph); + dictionaryBuilder.append('\t').append(homograph); lookupKeys.add(homograph); ambiguousLookupKeys.add(homograph); diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java new file mode 100644 index 0000000..1145936 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java @@ -0,0 +1,185 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.util.Objects; + +/** + * Reusable deterministic token sequence for benchmark-only token streams. + * + *

    + * The sequence keeps stable token ordering and offset progression while avoiding + * per-token object creation during iteration. + *

    + */ +final class BenchmarkTokenSequence { + + /** + * Shared backing corpus as character arrays. + */ + private char[][] tokenCharacters; + + /** + * Number of active tokens in the sequence. + */ + private int tokenCount; + + /** + * Cursor index for the currently emitted token. + */ + private int cursor; + + /** + * Current token character array. + */ + private char[] currentToken; + + /** + * Start offset of the current token. + */ + private int currentStartOffset; + + /** + * End offset of the current token. + */ + private int currentEndOffset; + + /** + * Offset of the next token start. + */ + private int nextOffset; + + /** + * Creates a reusable token sequence. + * + * @param tokens token corpus source + */ + BenchmarkTokenSequence(final String[] tokens) { + setTokens(tokens); + } + + /** + * Sets a new token corpus for this sequence. + * + *

    + * The sequence stores copied character arrays so token reads can be reused + * without creating per-token objects during benchmark iteration. + *

    + * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + Objects.requireNonNull(tokens, "tokens"); + this.tokenCharacters = new char[tokens.length][]; + for (int index = 0; index < tokens.length; index++) { + final String token = Objects.requireNonNull(tokens[index], "tokens[" + index + "]"); + this.tokenCharacters[index] = token.toCharArray(); + } + + this.tokenCount = this.tokenCharacters.length; + reset(); + } + + /** + * Resets stream position for reuse. + */ + void reset() { + this.cursor = 0; + this.nextOffset = 0; + this.currentStartOffset = 0; + this.currentEndOffset = 0; + this.currentToken = null; + } + + /** + * Returns whether at least one token remains in the sequence. + * + * @return true if a token can be emitted + */ + boolean hasNext() { + return this.cursor < this.tokenCount; + } + + /** + * Advances to the next token. + * + * @return true if a token was emitted + */ + boolean advance() { + if (!hasNext()) { + return false; + } + + final char[] token = this.tokenCharacters[this.cursor]; + this.currentToken = token; + this.currentStartOffset = this.nextOffset; + this.currentEndOffset = this.currentStartOffset + token.length; + this.nextOffset = this.currentEndOffset + 1; + this.cursor++; + return true; + } + + /** + * Returns the current token in the sequence. + * + * @return current token character array + */ + char[] currentToken() { + return this.currentToken; + } + + /** + * Returns current token start offset for token stream attributes. + * + * @return start offset + */ + int currentStartOffset() { + return this.currentStartOffset; + } + + /** + * Returns current token end offset for token stream attributes. + * + * @return end offset + */ + int currentEndOffset() { + return this.currentEndOffset; + } + + /** + * Returns final stream offset value used by {@code end()}. + * + * @return final offset + */ + int endOffset() { + return this.nextOffset > 0 ? this.nextOffset - 1 : 0; + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java new file mode 100644 index 0000000..60fb354 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java @@ -0,0 +1,146 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +/** + * Reusable Lucene {@link TokenStream} backed by a deterministic token array. + * + *

    + * Instances are mutable and intended for one JMH worker thread. The stream + * copies configured token text into reusable character storage so benchmark + * iteration can replay the same token sequence without mutating the shared + * source array. + *

    + */ +final class BenchmarkTokenStream extends TokenStream { + + /** + * Current token text attribute. + */ + private final CharTermAttribute charTermAttribute; + + /** + * Offset attribute used by Lucene filters that inspect offsets. + */ + private final OffsetAttribute offsetAttribute; + + /** + * Position increment attribute for one-token-at-a-time streams. + */ + private final PositionIncrementAttribute positionIncrementAttribute; + + /** + * Reusable token sequence. + */ + private final BenchmarkTokenSequence tokenSequence; + + /** + * Creates a stream over the supplied tokens. + * + * @param tokens initial token corpus + */ + BenchmarkTokenStream(final String[] tokens) { + this.tokenSequence = new BenchmarkTokenSequence(tokens); + this.charTermAttribute = addAttribute(CharTermAttribute.class); + this.offsetAttribute = addAttribute(OffsetAttribute.class); + this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + } + + /** + * Replaces the configured token corpus. + * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + this.tokenSequence.setTokens(tokens); + } + + /** + * Returns whether all configured tokens have been emitted. + * + * @return {@code true} after the current pass is exhausted + */ + boolean isDrained() { + return !this.tokenSequence.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean incrementToken() throws IOException { + if (!this.tokenSequence.advance()) { + return false; + } + + clearAttributes(); + final char[] token = this.tokenSequence.currentToken(); + this.charTermAttribute.copyBuffer(token, 0, token.length); + this.positionIncrementAttribute.setPositionIncrement(1); + this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset()); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public void reset() throws IOException { + super.reset(); + this.tokenSequence.reset(); + } + + /** + * {@inheritDoc} + */ + @Override + public void end() throws IOException { + super.end(); + final int endOffset = this.tokenSequence.endOffset(); + this.offsetAttribute.setOffset(endOffset, endOffset); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + super.close(); + this.charTermAttribute.setEmpty(); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java index bb99a19..0c302f4 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java @@ -30,43 +30,24 @@ ******************************************************************************/ package org.egothor.stemmer.benchmark; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; +import java.io.IOException; + +import org.egothor.stemmer.StemmerPatchTrieLoader; /** * Builds a deterministic English token corpus for side-by-side stemming - * benchmarks. + * benchmarks from the bundled Radixor English dictionary resource. * *

    - * The generated corpus mixes: - *

    - *
      - *
    • simple inflections
      • - *
    • common derivational forms
      • - *
    • US/UK spelling families
      • - *
    • forms that are suitable for comparison against the bundled - * {@code US_UK_PROFI} Radixor dictionary
      • - *
    - * - *

    - * The goal is not to simulate natural language frequency distribution exactly, - * but to provide a stable and reproducible comparison workload for benchmark - * runs and regression tracking. + * The dictionary resource stores the expected stem as the first tab-separated + * field on each line and its surface variants on the same line. This helper + * uses only token/root pairs where the token differs from the expected root for + * timing. Resources smaller than the shared timing minimum are repeated + * deterministically by {@link LanguageBenchmarkCorpus}. *

    */ final class EnglishComparisonCorpus { - /** - * Canonical lexical bases used to generate the token workload. - */ - private static final String[] BASES = { "analyze", "analyse", "color", "colour", "center", "centre", "organize", - "organise", "optimize", "optimise", "characterize", "characterise", "connect", "construct", "compute", - "design", "develop", "engineer", "govern", "improve", "index", "inform", "manage", "model", "observe", - "operate", "perform", "predict", "prepare", "process", "project", "protect", "publish", "query", "reduce", - "refresh", "render", "resolve", "return", "search", "select", "signal", "store", "structure", "support", - "transform", "update", "validate", "value" }; - /** * Utility class. */ @@ -77,64 +58,21 @@ final class EnglishComparisonCorpus { /** * Creates a deterministic token corpus for English stemming comparison. * - * @param familyCount number of generated lexical families * @return token array in stable order + * @throws IOException if the bundled English resource cannot be read */ - static String[] createTokens(final int familyCount) { - if (familyCount < 1) { - throw new IllegalArgumentException("familyCount must be at least 1."); - } - - final List tokens = new ArrayList<>(familyCount * 14); - - for (int index = 0; index < familyCount; index++) { - final String base = createBase(index); - - tokens.add(base); - tokens.add(base + "s"); - tokens.add(base + "ed"); - tokens.add(base + "ing"); - tokens.add(base + "er"); - tokens.add(base + "ers"); - tokens.add(base + "ly"); - tokens.add(base + "ness"); - tokens.add(base + "ment"); - tokens.add(base + "ments"); - tokens.add(base + "able"); - tokens.add(base + "ability"); - - if (base.endsWith("ize")) { - tokens.add(base.substring(0, base.length() - 3) + "isation"); - tokens.add(base.substring(0, base.length() - 3) + "ised"); - } - - if (base.endsWith("ise")) { - tokens.add(base.substring(0, base.length() - 3) + "ization"); - tokens.add(base.substring(0, base.length() - 3) + "ized"); - } - } - - return tokens.toArray(String[]::new); + static String[] createTokens() throws IOException { + return createCorpus().tokens(); } /** - * Creates one deterministic base token. + * Creates a deterministic changed-token corpus and expected root array for + * English stemming comparison. * - * @param index base ordinal - * @return generated lexical base + * @return token corpus with expected roots + * @throws IOException if the bundled English resource cannot be read */ - private static String createBase(final int index) { - return (BASES[index % BASES.length] + suffix(index)).toLowerCase(Locale.ROOT); - } - - /** - * Creates a compact discriminator suffix so that large corpora remain unique - * while retaining stable lexical families. - * - * @param value ordinal value - * @return compact discriminator - */ - private static String suffix(final int value) { - return Integer.toString(value, Character.MAX_RADIX); + static LanguageBenchmarkCorpus.Corpus createCorpus() throws IOException { + return LanguageBenchmarkCorpus.createChangedCorpus(StemmerPatchTrieLoader.Language.US_UK); } } diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java new file mode 100644 index 0000000..d626cb1 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java @@ -0,0 +1,431 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.FrequencyTrieBuilders; +import org.egothor.stemmer.PatchCommandEncoder; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.ReductionSettings; +import org.egothor.stemmer.StemmerDictionaryParser; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.WordTraversalDirection; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures Radixor English stemming quality and changed-token speed when the + * runtime trie is trained from a deterministic percentage of dictionary rows. + * + *

    + * The measured stemmer always uses {@link CompiledPatchCommand} values. Quality + * is evaluated against the complete English dictionary corpus, while speed is + * measured over the complete changed-token English corpus used by the comparison + * benchmarks. + *

    + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class EnglishRadixorDictionaryCoverageBenchmark { + + /** + * Shared benchmark state for one dictionary-row coverage percentage. + */ + @State(Scope.Benchmark) + public static class CoverageState { + + /** + * Percentage of parsed English dictionary rows used to build the Radixor trie. + */ + @Param({ "100", "90", "80", "70", "60", "50", "40", "30", "20", "10" }) + public int coveragePercent; + + /** + * Full English corpus used for exact-root accounting. + */ + private LanguageBenchmarkCorpus.Corpus fullCorpus; + + /** + * Complete changed-token English corpus used for speed measurement. + */ + private LanguageBenchmarkCorpus.Corpus changedCorpus; + + /** + * Radixor stemmer backed by a trie built from selected dictionary rows. + */ + private RadixorBenchmarkStemmer stemmer; + + /** + * Parsed dictionary row count before deterministic coverage selection. + */ + private int totalRowCount; + + /** + * Selected dictionary row count for the configured coverage percentage. + */ + private int selectedRowCount; + + /** + * Builds the reduced dictionary trie and shared corpora before measurement. + * + * @throws IOException if the English dictionary resource cannot be read + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final List rows = readEnglishRows(); + this.totalRowCount = rows.size(); + final List selectedRows = selectRows(rows, this.coveragePercent); + this.selectedRowCount = selectedRows.size(); + this.fullCorpus = LanguageBenchmarkCorpus.createFullCorpus(StemmerPatchTrieLoader.Language.US_UK); + this.changedCorpus = LanguageBenchmarkCorpus.createChangedCorpus(StemmerPatchTrieLoader.Language.US_UK); + this.stemmer = new RadixorBenchmarkStemmer(buildCompiledTrie(selectedRows)); + } + } + + /** + * JMH auxiliary counters for dictionary-row coverage and exact-root agreement. + */ + @State(Scope.Thread) + @AuxCounters(AuxCounters.Type.EVENTS) + public static class CoverageCounters { + + /** + * Number of exact output/root matches over the full dictionary corpus. + */ + public long correctMatches; + + /** + * Number of evaluated tokens over the full dictionary corpus. + */ + public long evaluatedTokens; + + /** + * Number of exact output/root matches where token and root differ. + */ + public long changedCorrectMatches; + + /** + * Number of evaluated tokens where token and root differ. + */ + public long changedEvaluatedTokens; + + /** + * Number of exact output/root matches where token already equals root. + */ + public long rootPreservedMatches; + + /** + * Number of evaluated tokens where token already equals root. + */ + public long rootEvaluatedTokens; + + /** + * Number of parsed dictionary rows used for trie construction. + */ + public long selectedRows; + + /** + * Total number of parsed dictionary rows available. + */ + public long totalRows; + + /** + * Resets counters before each measured iteration. + */ + @Setup(Level.Iteration) + public void reset() { + this.correctMatches = 0L; + this.evaluatedTokens = 0L; + this.changedCorrectMatches = 0L; + this.changedEvaluatedTokens = 0L; + this.rootPreservedMatches = 0L; + this.rootEvaluatedTokens = 0L; + this.selectedRows = 0L; + this.totalRows = 0L; + } + } + + /** + * Measures direct Radixor stemming over the complete English changed-token + * corpus. + * + * @param state shared coverage state + * @param blackhole result sink + */ + @Benchmark + public void changedTokenStemmingSpeed(final CoverageState state, final Blackhole blackhole) { + final String[] tokens = state.changedCorpus.tokens(); + final RadixorBenchmarkStemmer stemmer = state.stemmer; + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures exact-root agreement over the complete English dictionary corpus. + * + * @param state shared coverage state + * @param counters auxiliary exact-root counters + * @param blackhole result sink + * @return exact-root match count for one benchmark operation + */ + @Benchmark + public int exactRootAgreement(final CoverageState state, final CoverageCounters counters, + final Blackhole blackhole) { + final QualityCounts counts = evaluate(state.fullCorpus, state.stemmer, blackhole); + counters.correctMatches += counts.correctMatches(); + counters.evaluatedTokens += counts.evaluatedTokens(); + counters.changedCorrectMatches += counts.changedCorrectMatches(); + counters.changedEvaluatedTokens += counts.changedEvaluatedTokens(); + counters.rootPreservedMatches += counts.rootPreservedMatches(); + counters.rootEvaluatedTokens += counts.rootEvaluatedTokens(); + counters.selectedRows += state.selectedRowCount; + counters.totalRows += state.totalRowCount; + return counts.correctMatches(); + } + + private static QualityCounts evaluate(final LanguageBenchmarkCorpus.Corpus corpus, + final RadixorBenchmarkStemmer stemmer, final Blackhole blackhole) { + final String[] tokens = corpus.tokens(); + final String[] roots = corpus.expectedRoots(); + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + for (int index = 0; index < tokens.length; index++) { + final String token = tokens[index]; + final String root = roots[index]; + final String actual = stemmer.stem(token); + blackhole.consume(actual); + final boolean exact = Objects.equals(root, actual); + if (exact) { + correct++; + } + if (Objects.equals(token, root)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + return new QualityCounts(correct, tokens.length, changedCorrect, changedEvaluated, rootPreserved, + rootEvaluated); + } + + private static FrequencyTrie buildCompiledTrie(final List rows) { + final ReductionSettings settings = new ReductionSettings( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, + ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT, + ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO, + true); + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder<>(String[]::new, settings, + WordTraversalDirection.BACKWARD); + final PatchCommandEncoder encoder = PatchCommandEncoder.builder() + .traversalDirection(WordTraversalDirection.BACKWARD) + .build(); + + for (DictionaryRow row : rows) { + builder.put(row.stem(), encoder.encode(row.stem(), row.stem())); + for (String variant : row.variants()) { + if (!variant.equals(row.stem())) { + builder.put(variant, encoder.encode(variant, row.stem())); + } + } + } + + final FrequencyTrie trie = builder.build(); + final Map compiledCommands = new HashMap(4096); + return FrequencyTrieBuilders.mapValues(trie, CompiledPatchCommand[]::new, trie.metadata().reductionSettings(), + patch -> compiledCommands.computeIfAbsent(patch, + value -> CompiledPatchCommand.compile(value, trie.traversalDirection()))); + } + + private static List selectRows(final List rows, final int coveragePercent) { + if (coveragePercent < 1 || coveragePercent > 100) { + throw new IllegalArgumentException("coveragePercent must be between 1 and 100."); + } + if (coveragePercent == 100) { + return List.copyOf(rows); + } + + final int selectedCount = Math.max(1, Math.round(rows.size() * coveragePercent / 100.0F)); + final List rankedRows = new ArrayList(rows); + rankedRows.sort(Comparator.comparingLong(DictionaryRow::rank).thenComparingInt(DictionaryRow::lineNumber)); + + final Set selectedLineNumbers = new HashSet(selectedCount); + for (int index = 0; index < selectedCount; index++) { + selectedLineNumbers.add(rankedRows.get(index).lineNumber()); + } + + final List selectedRows = new ArrayList(selectedCount); + for (DictionaryRow row : rows) { + if (selectedLineNumbers.contains(row.lineNumber())) { + selectedRows.add(row); + } + } + return selectedRows; + } + + private static List readEnglishRows() throws IOException { + final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath(); + final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath); + if (resource == null) { + throw new IllegalStateException("Missing bundled English dictionary resource " + resourcePath + "."); + } + + final List rows = new ArrayList(400_000); + try (InputStream inputStream = resource; + GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); + InputStreamReader inputStreamReader = new InputStreamReader(gzipInputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(inputStreamReader)) { + StemmerDictionaryParser.parse(reader, resourcePath, (stem, variants, lineNumber) -> { + rows.add(new DictionaryRow(lineNumber, stem, variants, rank(lineNumber, stem, variants))); + }); + } + return rows; + } + + private static long rank(final int lineNumber, final String stem, final String[] variants) { + long hash = 0xcbf29ce484222325L; + hash = mix(hash, lineNumber); + hash = mix(hash, stem); + for (String variant : variants) { + hash = mix(hash, variant); + } + return hash; + } + + private static long mix(final long hash, final int value) { + long result = hash; + result ^= value & 0xFFL; + result *= 0x100000001b3L; + result ^= value >>> 8 & 0xFFL; + result *= 0x100000001b3L; + result ^= value >>> 16 & 0xFFL; + result *= 0x100000001b3L; + result ^= value >>> 24 & 0xFFL; + result *= 0x100000001b3L; + return result; + } + + private static long mix(final long hash, final String value) { + long result = hash; + for (int index = 0; index < value.length(); index++) { + final char character = value.charAt(index); + result ^= character & 0xFFL; + result *= 0x100000001b3L; + result ^= character >>> 8; + result *= 0x100000001b3L; + } + result ^= 0xFFL; + result *= 0x100000001b3L; + return result; + } + + /** + * One parsed dictionary row with deterministic selection rank. + * + * @param lineNumber source dictionary line number + * @param stem canonical stem from the first column + * @param variants normalized variants from following columns + * @param rank deterministic selection rank + */ + private record DictionaryRow(int lineNumber, String stem, String[] variants, long rank) { + + /** + * Creates one immutable dictionary row snapshot. + * + * @param lineNumber source dictionary line number + * @param stem canonical stem from the first column + * @param variants normalized variants from following columns + * @param rank deterministic selection rank + */ + DictionaryRow { + Objects.requireNonNull(stem, "stem"); + variants = variants.clone(); + } + + @Override + public String[] variants() { + return this.variants.clone(); + } + } + + /** + * Exact-root accounting result for one quality operation. + * + * @param correctMatches exact-root matches for all tokens + * @param evaluatedTokens evaluated token count + * @param changedCorrectMatches exact-root matches for changed tokens + * @param changedEvaluatedTokens evaluated changed-token count + * @param rootPreservedMatches exact-root matches for root-equal tokens + * @param rootEvaluatedTokens evaluated root-equal token count + */ + private record QualityCounts(int correctMatches, int evaluatedTokens, int changedCorrectMatches, + int changedEvaluatedTokens, int rootPreservedMatches, int rootEvaluatedTokens) { + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java index aef1d87..5692112 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java @@ -32,140 +32,314 @@ package org.egothor.stemmer.benchmark; import java.io.IOException; import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.en.EnglishMinimalStemFilter; +import org.apache.lucene.analysis.en.EnglishPossessiveFilter; +import org.apache.lucene.analysis.en.KStemFilter; +import org.apache.lucene.analysis.en.PorterStemFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.egothor.stemmer.benchmark.snowball.ext.englishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer; -import org.egothor.stemmer.FrequencyTrie; -import org.egothor.stemmer.PatchCommandEncoder; -import org.egothor.stemmer.ReductionMode; -import org.egothor.stemmer.StemmerPatchTrieLoader; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; -import org.tartarus.snowball.ext.englishStemmer; -import org.tartarus.snowball.ext.porterStemmer; + +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.StemmerDictionaryParser; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; /** - * Compares English stemming throughput across Radixor and Snowball stemmers. + * Compares English stemming throughput across Radixor and selected Java + * algorithm paths with a shared deterministic corpus. * *

    - * The benchmark processes the same deterministic token array with: + * The comparison uses one shared changed-token dictionary array for all methods: *

    *
      - *
    • Radixor using bundled {@link StemmerPatchTrieLoader.Language#US_UK}
      • - *
    • Snowball original Porter stemmer
      • - *
    • Snowball English stemmer, commonly referred to as Porter2
      • + *
    • Radixor direct dictionary lookup
      • + *
    • Snowball Porter
      • + *
    • Snowball English (Porter2)
      • + *
    • Lucene direct Porter API (generated copy)
      • + *
    • Lucene Porter, KStem, and EnglishMinimal token-filter paths
      • + *
    • Benchmark-only Paice/Husk Lancaster baseline
      • *
    - * - *

    - * This benchmark compares throughput on a shared workload. It does not imply - * that the algorithms are linguistically equivalent. - *

    */ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 1) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) public class EnglishStemmerComparisonBenchmark { /** - * Shared benchmark data. + * Shared, parameterized benchmark corpus state. */ @State(Scope.Benchmark) public static class SharedState { /** - * Number of generated lexical families. - */ - @Param({ "1000", "5000" }) - public int familyCount; - - /** - * Token workload processed by all compared stemmers. + * Shared deterministic token corpus. */ private String[] tokens; /** - * Radixor trie loaded from the bundled professional English dictionary. + * Radixor benchmark adapter for the US/UK benchmark corpus. */ - private FrequencyTrie radixorTrie; + private RadixorBenchmarkStemmer radixorStemmer; /** - * Initializes the shared benchmark state. - * - * @throws IOException if the bundled Radixor dictionary cannot be loaded + * Initializes shared corpus and trie state once per trial. */ @Setup(Level.Trial) - public void setUp() throws IOException { - this.tokens = EnglishComparisonCorpus.createTokens(this.familyCount); - this.radixorTrie = StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK, true, - ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); + public void setUp() throws java.io.IOException { + Logger.getLogger(StemmerDictionaryParser.class.getName()) + .setLevel(java.util.logging.Level.OFF); + Logger.getLogger(StemmerDictionaryParser.class.getName()).setUseParentHandlers(false); + this.tokens = EnglishComparisonCorpus.createTokens(); + this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); } } /** - * Per-thread reusable Snowball stemmers. + * Reusable direct stemmer instances. */ @State(Scope.Thread) - public static class SnowballState { + public static class DirectStemmerState { /** - * Adapter for the original Porter stemmer. + * Snowball classic Porter. */ private SnowballStemmerAdapter porterStemmer; /** - * Adapter for the Snowball English stemmer. + * Snowball English (Porter2) for legacy and dictionary comparison. */ - private SnowballStemmerAdapter englishStemmer; + private SnowballStemmerAdapter englishPorterStemmer; /** - * Initializes reusable Snowball stemmers for the executing thread. + * Generated Lucene direct Porter implementation copy. + */ + private LucenePorterStemmerCopied lucenePorter; + + /** + * Benchmark-only Paice/Husk Lancaster implementation. + */ + private PaiceHuskLancasterStemmer paiceHuskLancaster; + + /** + * Apache OpenNLP Porter stemmer. + */ + private opennlp.tools.stemmer.PorterStemmer openNlpPorterStemmer; + + /** + * Initializes mutable stemmer instances reused by all benchmark calls. */ @Setup(Level.Trial) public void setUp() { this.porterStemmer = new SnowballStemmerAdapter(porterStemmer::new); - this.englishStemmer = new SnowballStemmerAdapter(englishStemmer::new); + this.englishPorterStemmer = new SnowballStemmerAdapter(englishStemmer::new); + this.lucenePorter = new LucenePorterStemmerCopied(); + this.paiceHuskLancaster = new PaiceHuskLancasterStemmer(); + this.openNlpPorterStemmer = new opennlp.tools.stemmer.PorterStemmer(); + } + } + + /** + * Reusable Lucene token streams and filters. + */ + @State(Scope.Thread) + public static class LuceneFilterState { + + /** + * Reused Porter filter stream input. + */ + private final EnglishStemmerComparisonTokenStream porterStemFilterInput; + + /** + * Porter token filter for public API integration-path comparison. + */ + private final PorterStemFilter porterStemFilter; + + /** + * Porter filter attributes. + */ + private final CharTermAttribute porterStemFilterTerm; + + /** + * Reused KStem filter stream input. + */ + private final EnglishStemmerComparisonTokenStream kStemFilterInput; + + /** + * KStem token filter for a second Lucene English baseline. + */ + private final KStemFilter kStemFilter; + + /** + * KStem filter attributes. + */ + private final CharTermAttribute kStemTerm; + + /** + * Reused minimal stem filter stream input. + */ + private final EnglishStemmerComparisonTokenStream englishMinimalStemFilterInput; + + /** + * EnglishMinimal token filter. + */ + private final EnglishMinimalStemFilter englishMinimalStemFilter; + + /** + * EnglishMinimal filter attributes. + */ + private final CharTermAttribute englishMinimalTerm; + + /** + * Reused English possessive filter stream input. + */ + private final EnglishStemmerComparisonTokenStream englishPossessiveFilterInput; + + /** + * English possessive filter. + */ + private final EnglishPossessiveFilter englishPossessiveFilter; + + /** + * English possessive filter attributes. + */ + private final CharTermAttribute englishPossessiveTerm; + + /** + * Creates benchmark stream/filter state and attaches token attributes. + */ + public LuceneFilterState() { + this.porterStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]); + this.porterStemFilter = new PorterStemFilter(this.porterStemFilterInput); + this.porterStemFilterTerm = this.porterStemFilter.getAttribute(CharTermAttribute.class); + + this.kStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]); + this.kStemFilter = new KStemFilter(this.kStemFilterInput); + this.kStemTerm = this.kStemFilter.getAttribute(CharTermAttribute.class); + + this.englishMinimalStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]); + this.englishMinimalStemFilter = new EnglishMinimalStemFilter(this.englishMinimalStemFilterInput); + this.englishMinimalTerm = this.englishMinimalStemFilter.getAttribute(CharTermAttribute.class); + + this.englishPossessiveFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]); + this.englishPossessiveFilter = new EnglishPossessiveFilter(this.englishPossessiveFilterInput); + this.englishPossessiveTerm = this.englishPossessiveFilter.getAttribute(CharTermAttribute.class); + } + + /** + * Rebinds the shared corpus and resets all streams for another measured + * operation. + * + *

    + * The {@code String[]} to Lucene character-buffer conversion is deliberately + * performed every time so TokenFilter benchmarks include the cost of adapting + * the benchmark's canonical string corpus to Lucene's mutable token + * attributes. + *

    + * + * @param tokens benchmark token corpus + */ + void configure(final String[] tokens) throws IOException { + this.porterStemFilterInput.setTokens(tokens); + this.kStemFilterInput.setTokens(tokens); + this.englishMinimalStemFilterInput.setTokens(tokens); + this.englishPossessiveFilterInput.setTokens(tokens); + + this.porterStemFilter.reset(); + this.kStemFilter.reset(); + this.englishMinimalStemFilter.reset(); + this.englishPossessiveFilter.reset(); + } + + /** + * Reuses one mutable filter stream and returns all emitted tokens to blackhole. + * + * @param stream benchmark token stream with configured filter + * @param term token text attribute + * @param blackhole sink + * @throws IOException on token stream failure + */ + private static void consume(final TokenStream stream, final CharTermAttribute term, final Blackhole blackhole) + throws IOException { + while (stream.incrementToken()) { + blackhole.consume(term.toString()); + } + stream.end(); + } + + /** + * Executes Porter filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runPorterStemFilter(final Blackhole blackhole) throws IOException { + consume(this.porterStemFilter, this.porterStemFilterTerm, blackhole); + } + + /** + * Executes KStem filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runKStemFilter(final Blackhole blackhole) throws IOException { + consume(this.kStemFilter, this.kStemTerm, blackhole); + } + + /** + * Executes English minimal filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runEnglishMinimalStemFilter(final Blackhole blackhole) throws IOException { + consume(this.englishMinimalStemFilter, this.englishMinimalTerm, blackhole); + } + + /** + * Executes English possessive filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runEnglishPossessiveFilter(final Blackhole blackhole) throws IOException { + consume(this.englishPossessiveFilter, this.englishPossessiveTerm, blackhole); } } /** * Measures Radixor preferred-result stemming throughput. * - * @param sharedState shared benchmark data - * @param blackhole sink preventing dead-code elimination + *

    + * This path uses a single shared dictionary lookup and patch application. + *

    + * + * @param sharedState shared corpus and trie + * @param blackhole result sink */ @Benchmark public void radixorUsUkProfiPreferredStem(final SharedState sharedState, final Blackhole blackhole) { final String[] tokens = sharedState.tokens; - final FrequencyTrie trie = sharedState.radixorTrie; - - for (String token : tokens) { - final String patch = trie.get(token); - final String stem = patch == null ? token : PatchCommandEncoder.apply(token, patch); - blackhole.consume(stem); - } - } - - /** - * Measures Snowball original Porter stemming throughput. - * - * @param sharedState shared benchmark data - * @param snowballState reusable Snowball stemmers - * @param blackhole sink preventing dead-code elimination - */ - @Benchmark - public void snowballOriginalPorter(final SharedState sharedState, final SnowballState snowballState, - final Blackhole blackhole) { - final String[] tokens = sharedState.tokens; - final SnowballStemmerAdapter stemmer = snowballState.porterStemmer; + final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer; for (String token : tokens) { blackhole.consume(stemmer.stem(token)); @@ -173,25 +347,169 @@ public class EnglishStemmerComparisonBenchmark { } /** - * Measures Snowball English stemming throughput. + * Measures the canonical Snowball Porter stemming throughput used by the + * performance badge. * *

    - * Snowball English is the newer English stemmer commonly referred to as - * Porter2. + * This uses Snowball classic Porter as a direct stemmer API call and includes + * no Lucene token stream integration overhead. *

    * - * @param sharedState shared benchmark data - * @param snowballState reusable Snowball stemmers - * @param blackhole sink preventing dead-code elimination + * @param sharedState shared corpus + * @param stemmerState reusable Snowball adapter state + * @param blackhole result sink */ @Benchmark - public void snowballEnglishPorter2(final SharedState sharedState, final SnowballState snowballState, + public void snowballOriginalPorter(final SharedState sharedState, final DirectStemmerState stemmerState, final Blackhole blackhole) { final String[] tokens = sharedState.tokens; - final SnowballStemmerAdapter stemmer = snowballState.englishStemmer; + final SnowballStemmerAdapter stemmer = stemmerState.porterStemmer; for (String token : tokens) { blackhole.consume(stemmer.stem(token)); } } + + /** + * Measures Snowball English (Porter2) direct API throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable Snowball adapter state + * @param blackhole result sink + */ + @Benchmark + public void snowballEnglishPorter2(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final SnowballStemmerAdapter stemmer = stemmerState.englishPorterStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Lucene generated Porter stemmer API throughput. + * + *

    + * This path is a generated copy of Lucene's package-private PorterStemmer + * class, compiled into the JMH source set only. + *

    + * + * @param sharedState shared corpus + * @param stemmerState reusable Lucene copied API state + * @param blackhole result sink + */ + @Benchmark + public void lucenePorterStemmerCopied(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final LucenePorterStemmerCopied stemmer = stemmerState.lucenePorter; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Lucene Porter token-filter integration throughput. + * + *

    + * This includes stream, reusable token attributes, and filter overhead and is + * not equivalent to a direct API stemmer call. + *

    + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void lucenePorterStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runPorterStemFilter(blackhole); + } + + /** + * Measures Lucene KStem integration-path throughput. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneKStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runKStemFilter(blackhole); + } + + /** + * Measures Lucene EnglishMinimal integration-path throughput. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneEnglishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runEnglishMinimalStemFilter(blackhole); + } + + /** + * Measures benchmark-only Paice/Husk Lancaster throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable Paice/Husk instance + * @param blackhole sink + */ + @Benchmark + public void paiceHuskLancaster(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final PaiceHuskLancasterStemmer stemmer = stemmerState.paiceHuskLancaster; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Apache OpenNLP Porter stemming throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable OpenNLP Porter instance + * @param blackhole sink + */ + @Benchmark + public void opennlpPorterStemmer(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final opennlp.tools.stemmer.PorterStemmer stemmer = stemmerState.openNlpPorterStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token).toString()); + } + } + + /** + * Measures Lucene EnglishPossessiveFilter as a narrow possessive-removal + * baseline. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneEnglishPossessiveFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runEnglishPossessiveFilter(blackhole); + } } diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java new file mode 100644 index 0000000..1fdad74 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java @@ -0,0 +1,258 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Emits exact-root agreement metrics for the canonical English badge pair. + * + *

    + * This class is deliberately named so the existing focused include pattern for + * English stemmer comparison benchmarks includes it. The benchmark methods are + * separate from throughput methods so equality checks do not contaminate timing + * scores. + *

    + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class EnglishStemmerComparisonBenchmarkQuality { + + /** + * Shared English quality corpus and stemmer state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Complete English resource-derived corpus. + */ + private LanguageBenchmarkCorpus.Corpus corpus; + + /** + * Compiled Radixor English trie. + */ + private RadixorBenchmarkStemmer radixorStemmer; + + /** + * Reusable Snowball Porter adapter. + */ + private SnowballStemmerAdapter porterStemmer; + + /** + * Initializes quality resources. + * + * @throws IOException if corpus or trie loading fails + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.corpus = LanguageBenchmarkCorpus.createFullCorpus(StemmerPatchTrieLoader.Language.US_UK); + this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + this.porterStemmer = new SnowballStemmerAdapter(porterStemmer::new); + } + } + + /** + * JMH auxiliary counters for exact-root agreement. + */ + @State(Scope.Thread) + @AuxCounters(AuxCounters.Type.EVENTS) + public static class AccuracyCounters { + + /** + * Number of exact-root matches. + */ + public long correctMatches; + + /** + * Number of evaluated tokens. + */ + public long evaluatedTokens; + + /** + * Number of exact-root matches where the input token differs from the + * expected root. + */ + public long changedCorrectMatches; + + /** + * Number of evaluated tokens where the input token differs from the expected + * root. + */ + public long changedEvaluatedTokens; + + /** + * Number of exact-root matches where the input token is already the expected + * root. + */ + public long rootPreservedMatches; + + /** + * Number of evaluated tokens where the input token is already the expected + * root. + */ + public long rootEvaluatedTokens; + + /** + * Resets counters before each measured iteration. + */ + @Setup(Level.Iteration) + public void reset() { + this.correctMatches = 0L; + this.evaluatedTokens = 0L; + this.changedCorrectMatches = 0L; + this.changedEvaluatedTokens = 0L; + this.rootPreservedMatches = 0L; + this.rootEvaluatedTokens = 0L; + } + } + + /** + * Evaluates exact-root agreement for the canonical Radixor badge method. + * + * @param sharedState shared English quality state + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + @Benchmark + public int radixorUsUkProfiPreferredStemAccuracy(final SharedState sharedState, + final AccuracyCounters counters, final Blackhole blackhole) { + return evaluate(sharedState.corpus, sharedState.radixorStemmer::stem, counters, blackhole); + } + + /** + * Evaluates exact-root agreement for the canonical Snowball Porter badge + * method. + * + * @param sharedState shared English quality state + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + @Benchmark + public int snowballOriginalPorterAccuracy(final SharedState sharedState, + final AccuracyCounters counters, final Blackhole blackhole) { + return evaluate(sharedState.corpus, sharedState.porterStemmer::stem, counters, blackhole); + } + + /** + * Evaluates one stemmer against the expected roots. + * + * @param corpus token/root corpus + * @param stemmer stemmer under evaluation + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + private static int evaluate(final LanguageBenchmarkCorpus.Corpus corpus, final Stemmer stemmer, + final AccuracyCounters counters, final Blackhole blackhole) { + Objects.requireNonNull(corpus, "corpus"); + Objects.requireNonNull(stemmer, "stemmer"); + + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + final String[] tokens = corpus.tokens(); + final String[] expectedRoots = corpus.expectedRoots(); + for (int index = 0; index < tokens.length; index++) { + final String token = tokens[index]; + final String expectedRoot = expectedRoots[index]; + final String actual = stemmer.stem(token); + blackhole.consume(actual); + final boolean exact = Objects.equals(expectedRoot, actual); + if (exact) { + correct++; + } + if (Objects.equals(token, expectedRoot)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + + counters.correctMatches += correct; + counters.evaluatedTokens += tokens.length; + counters.changedCorrectMatches += changedCorrect; + counters.changedEvaluatedTokens += changedEvaluated; + counters.rootPreservedMatches += rootPreserved; + counters.rootEvaluatedTokens += rootEvaluated; + return correct; + } + + /** + * Direct stemmer function. + */ + @FunctionalInterface + private interface Stemmer { + + /** + * Produces one stem. + * + * @param token input token + * @return produced stem + */ + String stem(String token); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java new file mode 100644 index 0000000..a5ced0f --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +/** + * Reusable token stream driven by a deterministic token corpus. + * + *

    + * The stream emits each token from a shared array and supports repeated + * {@link #reset()} + {@link #incrementToken()} cycles without per-token + * object allocation. + *

    + */ +final class EnglishStemmerComparisonTokenStream extends TokenStream { + + /** + * Current token text. + */ + private final CharTermAttribute charTermAttribute; + + /** + * Token offsets for benchmark stream compliance. + */ + private final OffsetAttribute offsetAttribute; + + /** + * Position increment attribute for benchmark stream compliance. + */ + private final PositionIncrementAttribute positionIncrementAttribute; + + /** + * Reusable token source. + */ + private final BenchmarkTokenSequence tokenSequence; + + /** + * Creates a deterministic token stream for benchmark reuse. + * + * @param tokens tokens emitted by the stream + */ + EnglishStemmerComparisonTokenStream(final String[] tokens) { + this.tokenSequence = new BenchmarkTokenSequence(tokens); + this.charTermAttribute = addAttribute(CharTermAttribute.class); + this.offsetAttribute = addAttribute(OffsetAttribute.class); + this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + } + + /** + * Replaces the token corpus for this stream. + * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + this.tokenSequence.setTokens(tokens); + } + + /** + * Returns whether the stream is drained and ready to be exhausted. + * + * @return true if all configured tokens were consumed + */ + boolean isDrained() { + return !this.tokenSequence.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean incrementToken() throws IOException { + if (!this.tokenSequence.advance()) { + return false; + } + + clearAttributes(); + final char[] token = this.tokenSequence.currentToken(); + this.charTermAttribute.copyBuffer(token, 0, token.length); + this.positionIncrementAttribute.setPositionIncrement(1); + this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset()); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public void reset() throws IOException { + super.reset(); + this.tokenSequence.reset(); + } + + /** + * {@inheritDoc} + */ + @Override + public void end() throws IOException { + super.end(); + final int endOffset = this.tokenSequence.endOffset(); + this.offsetAttribute.setOffset(endOffset, endOffset); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + super.close(); + this.charTermAttribute.setEmpty(); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java index ee382e7..fc4416d 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java @@ -65,6 +65,7 @@ import org.openjdk.jmh.infra.Blackhole; @OutputTimeUnit(TimeUnit.NANOSECONDS) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 1) +@SuppressWarnings("deprecation") public class FrequencyTrieLookupBenchmark { /** diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java b/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java new file mode 100644 index 0000000..fc5c9a4 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java @@ -0,0 +1,445 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Locale; +import java.util.Objects; +import java.util.zip.GZIPInputStream; + +import org.egothor.stemmer.StemmerPatchTrieLoader; + +/** + * Builds deterministic language-specific benchmark corpora from bundled + * Radixor dictionary resources. + * + *

    + * Corpus construction is setup work only. It is intentionally based on the same + * resource that backs the Radixor benchmark path so every competitor for a + * language consumes the same changed-token timing workload, while quality + * benchmarks can still use the complete dictionary workload. + *

    + */ +final class LanguageBenchmarkCorpus { + + /** + * Minimum token count for timing benchmark operations. + */ + static final int MINIMUM_TIMING_TOKEN_COUNT = 5_000; + + /** + * Shared timing corpora keyed by bundled Radixor language. + */ + private static final Map TIMING_CORPORA = + new EnumMap<>(StemmerPatchTrieLoader.Language.class); + + /** + * Shared changed-token timing corpora keyed by bundled Radixor language. + */ + private static final Map CHANGED_TIMING_CORPORA = + new EnumMap<>(StemmerPatchTrieLoader.Language.class); + + /** + * Shared complete corpora keyed by bundled Radixor language. + */ + private static final Map FULL_CORPORA = + new EnumMap<>(StemmerPatchTrieLoader.Language.class); + + /** + * Utility class. + */ + private LanguageBenchmarkCorpus() { + throw new AssertionError("No instances."); + } + + /** + * Creates a deterministic changed-token timing corpus from a bundled language + * dictionary. + * + *

    + * Only token/root pairs where the token differs from the expected root are + * included. Smaller changed-token resources are repeated in stable order until + * the timing corpus reaches 5,000 tokens. + *

    + * + * @param language bundled Radixor language + * @return token array containing changed-token dictionary entries, repeated + * only when the changed-token resource is smaller than 5,000 tokens + * @throws IOException if the resource cannot be read + */ + static String[] createTokens(final StemmerPatchTrieLoader.Language language) throws IOException { + return createChangedCorpus(language).tokens(); + } + + /** + * Creates a deterministic changed-token timing corpus from a bundled language + * dictionary. + * + * @param language bundled Radixor language + * @return changed-token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createChangedCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedChangedCorpus(language); + } + + /** + * Creates a deterministic full-dictionary timing corpus and expected root + * array from a bundled language dictionary. + * + * @param language bundled Radixor language + * @return token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedCorpus(TIMING_CORPORA, language, true); + } + + /** + * Creates a deterministic full-dictionary timing corpus and expected root + * array from a bundled language dictionary. + * + *

    + * The complete dictionary token sequence is used when it contains at least + * {@code minimumTokenCount} tokens. Smaller resources are repeated in stable + * order until the minimum is reached. + *

    + * + * @param language bundled Radixor language + * @param minimumTokenCount minimum token count for timing + * @return token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createCorpus(final StemmerPatchTrieLoader.Language language, final int minimumTokenCount) + throws IOException { + Objects.requireNonNull(language, "language"); + if (minimumTokenCount < 1) { + throw new IllegalArgumentException("minimumTokenCount must be at least 1."); + } + if (minimumTokenCount == MINIMUM_TIMING_TOKEN_COUNT) { + return createCorpus(language); + } + + return buildTimingCorpus(language, minimumTokenCount); + } + + /** + * Creates or returns the shared complete corpus for a bundled language. + * + * @param language bundled Radixor language + * @return complete token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedCorpus(FULL_CORPORA, language, false); + } + + /** + * Returns a cached corpus, creating it once per JVM when necessary. + * + * @param cache corpus cache + * @param language bundled Radixor language + * @param timing whether the timing-minimum corpus should be built + * @return cached corpus instance + * @throws IOException if the resource cannot be read + */ + private static Corpus cachedCorpus(final Map cache, + final StemmerPatchTrieLoader.Language language, final boolean timing) throws IOException { + Objects.requireNonNull(cache, "cache"); + Objects.requireNonNull(language, "language"); + + synchronized (LanguageBenchmarkCorpus.class) { + final Corpus existing = cache.get(language); + if (existing != null) { + return existing; + } + + final Corpus created = timing ? buildTimingCorpus(language, MINIMUM_TIMING_TOKEN_COUNT) + : buildFullCorpus(language); + cache.put(language, created); + return created; + } + } + + /** + * Returns a cached changed-token timing corpus, creating it once per JVM when + * necessary. + * + * @param language bundled Radixor language + * @return changed-token timing corpus + * @throws IOException if the resource cannot be read + */ + private static Corpus cachedChangedCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + Objects.requireNonNull(language, "language"); + + synchronized (LanguageBenchmarkCorpus.class) { + final Corpus existing = CHANGED_TIMING_CORPORA.get(language); + if (existing != null) { + return existing; + } + + final Corpus created = buildChangedTimingCorpus(language, MINIMUM_TIMING_TOKEN_COUNT); + CHANGED_TIMING_CORPORA.put(language, created); + return created; + } + } + + /** + * Builds a deterministic timing corpus from a bundled language dictionary. + * + * @param language bundled Radixor language + * @param minimumTokenCount minimum token count for timing + * @return token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + private static Corpus buildTimingCorpus(final StemmerPatchTrieLoader.Language language, final int minimumTokenCount) + throws IOException { + Objects.requireNonNull(language, "language"); + if (minimumTokenCount < 1) { + throw new IllegalArgumentException("minimumTokenCount must be at least 1."); + } + + final List candidates = readCandidates(language, Integer.MAX_VALUE); + if (candidates.isEmpty()) { + throw new IllegalStateException("No benchmark corpus tokens were available for " + language + "."); + } + + final int timingTokenCount = Math.max(candidates.size(), minimumTokenCount); + final String[] tokens = new String[timingTokenCount]; + final String[] expectedRoots = new String[timingTokenCount]; + for (int index = 0; index < tokens.length; index++) { + final Entry entry = candidates.get(index % candidates.size()); + tokens[index] = entry.token(); + expectedRoots[index] = entry.root(); + } + return new Corpus(tokens, expectedRoots); + } + + /** + * Builds a deterministic changed-token timing corpus from a bundled language + * dictionary. + * + * @param language bundled Radixor language + * @param minimumTokenCount minimum token count for timing + * @return changed-token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + private static Corpus buildChangedTimingCorpus(final StemmerPatchTrieLoader.Language language, + final int minimumTokenCount) throws IOException { + Objects.requireNonNull(language, "language"); + if (minimumTokenCount < 1) { + throw new IllegalArgumentException("minimumTokenCount must be at least 1."); + } + + final List allCandidates = readCandidates(language, Integer.MAX_VALUE); + final List changedCandidates = new ArrayList<>(allCandidates.size()); + for (Entry entry : allCandidates) { + if (!Objects.equals(entry.token(), entry.root())) { + changedCandidates.add(entry); + } + } + if (changedCandidates.isEmpty()) { + throw new IllegalStateException("No changed-token benchmark corpus tokens were available for " + + language + "."); + } + + final int timingTokenCount = Math.max(changedCandidates.size(), minimumTokenCount); + final String[] tokens = new String[timingTokenCount]; + final String[] expectedRoots = new String[timingTokenCount]; + for (int index = 0; index < tokens.length; index++) { + final Entry entry = changedCandidates.get(index % changedCandidates.size()); + tokens[index] = entry.token(); + expectedRoots[index] = entry.root(); + } + return new Corpus(tokens, expectedRoots); + } + + /** + * Creates a complete deterministic token corpus and expected root array from a + * bundled language dictionary. + * + *

    + * This method is intended for exact-root quality accounting. It includes all + * single-token fields available in the dictionary resource and does not repeat + * small dictionaries to the timing minimum. + *

    + * + * @param language bundled Radixor language + * @return complete token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + private static Corpus buildFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + Objects.requireNonNull(language, "language"); + + final List candidates = readCandidates(language, Integer.MAX_VALUE); + if (candidates.isEmpty()) { + throw new IllegalStateException("No benchmark corpus tokens were available for " + language + "."); + } + + final String[] tokens = new String[candidates.size()]; + final String[] expectedRoots = new String[candidates.size()]; + for (int index = 0; index < tokens.length; index++) { + final Entry entry = candidates.get(index); + tokens[index] = entry.token(); + expectedRoots[index] = entry.root(); + } + return new Corpus(tokens, expectedRoots); + } + + /** + * Reads token candidates from a bundled compressed dictionary. + * + * @param language bundled Radixor language + * @param maximumTokenCount maximum token count to read + * @return deterministic candidate list + * @throws IOException if the resource cannot be read + */ + private static List readCandidates(final StemmerPatchTrieLoader.Language language, final int maximumTokenCount) + throws IOException { + final String resourcePath = language.resourcePath(); + final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath); + if (resource == null) { + throw new IllegalStateException("Missing bundled benchmark resource " + resourcePath + "."); + } + + final List candidates = new ArrayList<>(MINIMUM_TIMING_TOKEN_COUNT); + try (InputStream inputStream = resource; + GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); + InputStreamReader inputStreamReader = new InputStreamReader(gzipInputStream, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(inputStreamReader)) { + String line = reader.readLine(); + while (line != null && candidates.size() < maximumTokenCount) { + collectLineCandidates(line, candidates, maximumTokenCount); + line = reader.readLine(); + } + } + return candidates; + } + + /** + * Collects lower-case token candidates from one dictionary line. + * + * @param line dictionary line + * @param candidates mutable candidate list + * @param maximumTokenCount maximum token count to read + */ + private static void collectLineCandidates(final String line, final List candidates, + final int maximumTokenCount) { + if (line == null || line.isBlank() || line.startsWith("#") || line.startsWith("//")) { + return; + } + + final String[] fields = line.split("\t"); + if (fields.length == 0) { + return; + } + + final String root = normalizeToken(fields[0]); + if (root.isEmpty() || containsWhitespace(root)) { + return; + } + + for (String field : fields) { + if (candidates.size() >= maximumTokenCount) { + return; + } + final String token = normalizeToken(field); + if (!token.isEmpty() && !containsWhitespace(token)) { + candidates.add(new Entry(token, root)); + } + } + } + + /** + * Normalizes dictionary token text for deterministic benchmark lookup. + * + * @param token dictionary token field + * @return normalized token + */ + private static String normalizeToken(final String token) { + return token.trim().toLowerCase(Locale.ROOT); + } + + /** + * Returns whether a token contains Unicode whitespace. + * + * @param token token candidate + * @return {@code true} when whitespace is present + */ + private static boolean containsWhitespace(final String token) { + for (int index = 0; index < token.length(); index++) { + if (Character.isWhitespace(token.charAt(index))) { + return true; + } + } + return false; + } + + /** + * Immutable token corpus with expected roots. + * + * @param tokens benchmark token corpus + * @param expectedRoots expected root for each token + */ + record Corpus(String[] tokens, String[] expectedRoots) { + + /** + * Creates corpus data. + * + * @param tokens benchmark token corpus + * @param expectedRoots expected root for each token + */ + Corpus { + Objects.requireNonNull(tokens, "tokens"); + Objects.requireNonNull(expectedRoots, "expectedRoots"); + if (tokens.length != expectedRoots.length) { + throw new IllegalArgumentException("tokens and expectedRoots must have the same length."); + } + } + } + + /** + * Immutable dictionary-derived token/root entry. + * + * @param token token form + * @param root expected root + */ + private record Entry(String token, String root) { + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java new file mode 100644 index 0000000..d3be956 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java @@ -0,0 +1,1114 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.ar.ArabicNormalizationFilter; +import org.apache.lucene.analysis.core.DecimalDigitFilter; +import org.apache.lucene.analysis.cz.CzechStemFilter; +import org.apache.lucene.analysis.de.GermanLightStemFilter; +import org.apache.lucene.analysis.de.GermanMinimalStemFilter; +import org.apache.lucene.analysis.de.GermanNormalizationFilter; +import org.apache.lucene.analysis.de.GermanStemFilter; +import org.apache.lucene.analysis.es.SpanishLightStemFilter; +import org.apache.lucene.analysis.es.SpanishMinimalStemFilter; +import org.apache.lucene.analysis.es.SpanishPluralStemFilter; +import org.apache.lucene.analysis.fa.PersianNormalizationFilter; +import org.apache.lucene.analysis.fa.PersianStemFilter; +import org.apache.lucene.analysis.fi.FinnishLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter; +import org.apache.lucene.analysis.hu.HungarianLightStemFilter; +import org.apache.lucene.analysis.it.ItalianLightStemFilter; +import org.apache.lucene.analysis.morfologik.MorfologikFilter; +import org.apache.lucene.analysis.no.NorwegianLightStemFilter; +import org.apache.lucene.analysis.no.NorwegianMinimalStemFilter; +import org.apache.lucene.analysis.pl.PolishAnalyzer; +import org.apache.lucene.analysis.pt.PortugueseLightStemFilter; +import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter; +import org.apache.lucene.analysis.pt.PortugueseStemFilter; +import org.apache.lucene.analysis.ru.RussianLightStemFilter; +import org.apache.lucene.analysis.stempel.StempelFilter; +import org.apache.lucene.analysis.stempel.StempelStemmer; +import org.apache.lucene.analysis.sv.SwedishLightStemFilter; +import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import morfologik.stemming.Dictionary; +import morfologik.stemming.DictionaryLookup; +import morfologik.stemming.WordData; + +/** + * Compares Radixor with public Lucene stemmer integration paths for languages + * where both systems have a matching bundled resource. + * + *

    + * Each benchmark operation processes the same changed-token dictionary corpus + * for one language, repeated only when the changed-token resource contains + * fewer than 5,000 token fields. The token corpus is built during trial setup + * from Radixor's bundled dictionary for that same language. Lucene TokenFilter + * methods include TokenStream and attribute overhead; direct Stempel measures + * the public table-driven stemmer API without TokenFilter overhead. + *

    + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@SuppressWarnings("deprecation") +public class MultiLanguageStemmerComparisonBenchmark { + + /** + * Shared language corpus and Radixor trie state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Czech benchmark state. + */ + private LanguageState czech; + + /** + * German benchmark state. + */ + private LanguageState german; + + /** + * Spanish benchmark state. + */ + private LanguageState spanish; + + /** + * Persian benchmark state. + */ + private LanguageState persian; + + /** + * Finnish benchmark state. + */ + private LanguageState finnish; + + /** + * French benchmark state. + */ + private LanguageState french; + + /** + * Hungarian benchmark state. + */ + private LanguageState hungarian; + + /** + * Italian benchmark state. + */ + private LanguageState italian; + + /** + * Norwegian Bokmal benchmark state. + */ + private LanguageState norwegianBokmal; + + /** + * Polish benchmark state. + */ + private LanguageState polish; + + /** + * Portuguese benchmark state. + */ + private LanguageState portuguese; + + /** + * Russian benchmark state. + */ + private LanguageState russian; + + /** + * Swedish benchmark state. + */ + private LanguageState swedish; + + /** + * Ukrainian benchmark state. + */ + private LanguageState ukrainian; + + /** + * Initializes all language resources before measurement. + * + * @throws IOException if a bundled language resource cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.czech = load(StemmerPatchTrieLoader.Language.CS_CZ); + this.german = load(StemmerPatchTrieLoader.Language.DE_DE); + this.spanish = load(StemmerPatchTrieLoader.Language.ES_ES); + this.persian = load(StemmerPatchTrieLoader.Language.FA_IR); + this.finnish = load(StemmerPatchTrieLoader.Language.FI_FI); + this.french = load(StemmerPatchTrieLoader.Language.FR_FR); + this.hungarian = load(StemmerPatchTrieLoader.Language.HU_HU); + this.italian = load(StemmerPatchTrieLoader.Language.IT_IT); + this.norwegianBokmal = load(StemmerPatchTrieLoader.Language.NB_NO); + this.polish = load(StemmerPatchTrieLoader.Language.PL_PL); + this.portuguese = load(StemmerPatchTrieLoader.Language.PT_PT); + this.russian = load(StemmerPatchTrieLoader.Language.RU_RU); + this.swedish = load(StemmerPatchTrieLoader.Language.SV_SE); + this.ukrainian = load(StemmerPatchTrieLoader.Language.UK_UA); + } + } + + /** + * Per-thread Lucene filter state. + */ + @State(Scope.Thread) + public static class LuceneFilterState { + + /** + * Czech stem filter. + */ + private final FilterPipeline czechStem = new FilterPipeline( + input -> new CzechStemFilter(lowercase(input))); + + /** + * German classic stem filter. + */ + private final FilterPipeline germanStem = new FilterPipeline( + input -> new GermanStemFilter(lowercase(input))); + + /** + * German light stem filter. + */ + private final FilterPipeline germanLightStem = new FilterPipeline( + input -> new GermanLightStemFilter(germanNormalize(input))); + + /** + * German minimal stem filter. + */ + private final FilterPipeline germanMinimalStem = new FilterPipeline( + input -> new GermanMinimalStemFilter(germanNormalize(input))); + + /** + * Spanish light stem filter. + */ + private final FilterPipeline spanishLightStem = new FilterPipeline( + input -> new SpanishLightStemFilter(lowercase(input))); + + /** + * Spanish minimal stem filter. + */ + private final FilterPipeline spanishMinimalStem = new FilterPipeline( + input -> new SpanishMinimalStemFilter(lowercase(input))); + + /** + * Spanish plural stem filter. + */ + private final FilterPipeline spanishPluralStem = new FilterPipeline( + input -> new SpanishPluralStemFilter(lowercase(input))); + + /** + * Persian stem filter. + */ + private final FilterPipeline persianStem = new FilterPipeline( + input -> new PersianStemFilter(persianNormalize(input))); + + /** + * Finnish light stem filter. + */ + private final FilterPipeline finnishLightStem = new FilterPipeline( + input -> new FinnishLightStemFilter(lowercase(input))); + + /** + * French light stem filter. + */ + private final FilterPipeline frenchLightStem = new FilterPipeline( + input -> new FrenchLightStemFilter(lowercase(input))); + + /** + * French minimal stem filter. + */ + private final FilterPipeline frenchMinimalStem = new FilterPipeline( + input -> new FrenchMinimalStemFilter(lowercase(input))); + + /** + * Hungarian light stem filter. + */ + private final FilterPipeline hungarianLightStem = new FilterPipeline( + input -> new HungarianLightStemFilter(lowercase(input))); + + /** + * Italian light stem filter. + */ + private final FilterPipeline italianLightStem = new FilterPipeline( + input -> new ItalianLightStemFilter(lowercase(input))); + + /** + * Norwegian light stem filter. + */ + private final FilterPipeline norwegianLightStem = new FilterPipeline( + input -> new NorwegianLightStemFilter(lowercase(input))); + + /** + * Norwegian minimal stem filter. + */ + private final FilterPipeline norwegianMinimalStem = new FilterPipeline( + input -> new NorwegianMinimalStemFilter(lowercase(input))); + + /** + * Polish Stempel token filter. + */ + private final FilterPipeline polishStempelStem = new FilterPipeline( + input -> new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()))); + + /** + * Polish Morfologik token filter. + */ + private final FilterPipeline polishMorfologik = new FilterPipeline(MorfologikFilter::new); + + /** + * Portuguese full stem filter. + */ + private final FilterPipeline portugueseStem = new FilterPipeline( + input -> new PortugueseStemFilter(lowercase(input))); + + /** + * Portuguese light stem filter. + */ + private final FilterPipeline portugueseLightStem = new FilterPipeline( + input -> new PortugueseLightStemFilter(lowercase(input))); + + /** + * Portuguese minimal stem filter. + */ + private final FilterPipeline portugueseMinimalStem = new FilterPipeline( + input -> new PortugueseMinimalStemFilter(lowercase(input))); + + /** + * Russian light stem filter. + */ + private final FilterPipeline russianLightStem = new FilterPipeline( + input -> new RussianLightStemFilter(lowercase(input))); + + /** + * Swedish light stem filter. + */ + private final FilterPipeline swedishLightStem = new FilterPipeline( + input -> new SwedishLightStemFilter(lowercase(input))); + + /** + * Swedish minimal stem filter. + */ + private final FilterPipeline swedishMinimalStem = new FilterPipeline( + input -> new SwedishMinimalStemFilter(lowercase(input))); + + /** + * Ukrainian Morfologik token filter. + */ + private FilterPipeline ukrainianMorfologik; + + /** + * Initializes filter state that needs benchmark-only dictionary resources. + * + * @throws IOException if a benchmark-only dictionary cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final Dictionary dictionary = loadUkrainianMorfologikDictionary(); + this.ukrainianMorfologik = new FilterPipeline(input -> new MorfologikFilter(input, dictionary)); + } + } + + /** + * Per-thread direct non-TokenFilter stemmer state. + */ + @State(Scope.Thread) + public static class DirectState { + + /** + * Direct Stempel stemmer using Lucene's default Polish table. + */ + private StempelStemmer polishStempelStemmer; + + /** + * Direct Ukrainian Morfologik dictionary lookup. + */ + private DictionaryLookup ukrainianMorfologikLookup; + + /** + * Initializes direct stemmer instances before measurement. + * + * @throws IOException if a benchmark-only dictionary cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.polishStempelStemmer = new StempelStemmer(PolishAnalyzer.getDefaultTable()); + this.ukrainianMorfologikLookup = new DictionaryLookup(loadUkrainianMorfologikDictionary()); + } + } + + /** + * Runs Radixor over the Czech corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void czechRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.czech, blackhole); + } + + /** + * Runs Lucene CzechStemFilter over the Czech corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void czechLuceneCzechStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.czechStem.run(sharedState.czech.tokens, blackhole); + } + + /** + * Runs Radixor over the German corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void germanRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.german, blackhole); + } + + /** + * Runs Lucene GermanStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Lucene GermanLightStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanLightStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Lucene GermanMinimalStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanMinimalStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Radixor over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void spanishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.spanish, blackhole); + } + + /** + * Runs Lucene SpanishLightStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishLightStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Lucene SpanishMinimalStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishMinimalStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Lucene SpanishPluralStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishPluralStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishPluralStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Radixor over the Persian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void persianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.persian, blackhole); + } + + /** + * Runs Lucene PersianStemFilter over the Persian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void persianLucenePersianStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.persianStem.run(sharedState.persian.tokens, blackhole); + } + + /** + * Runs Radixor over the Finnish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void finnishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.finnish, blackhole); + } + + /** + * Runs Lucene FinnishLightStemFilter over the Finnish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void finnishLuceneFinnishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.finnishLightStem.run(sharedState.finnish.tokens, blackhole); + } + + /** + * Runs Radixor over the French corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void frenchRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.french, blackhole); + } + + /** + * Runs Lucene FrenchLightStemFilter over the French corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void frenchLuceneFrenchLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.frenchLightStem.run(sharedState.french.tokens, blackhole); + } + + /** + * Runs Lucene FrenchMinimalStemFilter over the French corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void frenchLuceneFrenchMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.frenchMinimalStem.run(sharedState.french.tokens, blackhole); + } + + /** + * Runs Radixor over the Hungarian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void hungarianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.hungarian, blackhole); + } + + /** + * Runs Lucene HungarianLightStemFilter over the Hungarian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void hungarianLuceneHungarianLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.hungarianLightStem.run(sharedState.hungarian.tokens, blackhole); + } + + /** + * Runs Radixor over the Italian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void italianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.italian, blackhole); + } + + /** + * Runs Lucene ItalianLightStemFilter over the Italian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void italianLuceneItalianLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.italianLightStem.run(sharedState.italian.tokens, blackhole); + } + + /** + * Runs Radixor over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void norwegianBokmalRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.norwegianBokmal, blackhole); + } + + /** + * Runs Lucene NorwegianLightStemFilter over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void norwegianBokmalLuceneNorwegianLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.norwegianLightStem.run(sharedState.norwegianBokmal.tokens, blackhole); + } + + /** + * Runs Lucene NorwegianMinimalStemFilter over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void norwegianBokmalLuceneNorwegianMinimalStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.norwegianMinimalStem.run(sharedState.norwegianBokmal.tokens, blackhole); + } + + /** + * Runs Radixor over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void polishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.polish, blackhole); + } + + /** + * Runs Lucene Stempel direct API over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct stemmer state + * @param blackhole result sink + */ + @Benchmark + public void polishLuceneStempelStemmerDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.polish.tokens; + final StempelStemmer stemmer = directState.polishStempelStemmer; + for (String token : tokens) { + final StringBuilder stem = stemmer.stem(token); + blackhole.consume(stem == null ? token : stem.toString()); + } + } + + /** + * Runs Lucene StempelFilter over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void polishLuceneStempelFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.polishStempelStem.run(sharedState.polish.tokens, blackhole); + } + + /** + * Runs Lucene MorfologikFilter over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void polishLuceneMorfologikFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.polishMorfologik.run(sharedState.polish.tokens, blackhole); + } + + /** + * Runs Radixor over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void portugueseRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.portuguese, blackhole); + } + + /** + * Runs Lucene PortugueseStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Lucene PortugueseLightStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseLightStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Lucene PortugueseMinimalStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseMinimalStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseMinimalStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Radixor over the Russian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void russianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.russian, blackhole); + } + + /** + * Runs Lucene RussianLightStemFilter over the Russian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void russianLuceneRussianLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.russianLightStem.run(sharedState.russian.tokens, blackhole); + } + + /** + * Runs Radixor over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void swedishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.swedish, blackhole); + } + + /** + * Runs Lucene SwedishLightStemFilter over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void swedishLuceneSwedishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.swedishLightStem.run(sharedState.swedish.tokens, blackhole); + } + + /** + * Runs Lucene SwedishMinimalStemFilter over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void swedishLuceneSwedishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.swedishMinimalStem.run(sharedState.swedish.tokens, blackhole); + } + + /** + * Runs Radixor over the Ukrainian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void ukrainianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.ukrainian, blackhole); + } + + /** + * Runs direct Morfologik Ukrainian dictionary lookup over the Ukrainian corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct stemmer state + * @param blackhole result sink + */ + @Benchmark + public void ukrainianMorfologikDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.ukrainian.tokens; + final DictionaryLookup lookup = directState.ukrainianMorfologikLookup; + for (String token : tokens) { + blackhole.consume(firstMorfologikStem(token, lookup)); + } + } + + /** + * Runs Lucene MorfologikFilter with the Ukrainian dictionary over the Ukrainian + * corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void ukrainianLuceneMorfologikFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.ukrainianMorfologik.run(sharedState.ukrainian.tokens, blackhole); + } + + /** + * Loads one language benchmark state. + * + * @param language bundled Radixor language + * @return initialized language state + * @throws IOException if the corpus or trie cannot be loaded + */ + private static LanguageState load(final StemmerPatchTrieLoader.Language language) throws IOException { + final String[] tokens = LanguageBenchmarkCorpus.createTokens(language); + final FrequencyTrie trie = StemmerPatchTrieLoader.loadCompiled(language, true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); + return new LanguageState(tokens, new RadixorBenchmarkStemmer(trie)); + } + + /** + * Runs Radixor direct lookup and patch application. + * + * @param languageState language state + * @param blackhole result sink + */ + private static void runRadixor(final LanguageState languageState, final Blackhole blackhole) { + final String[] tokens = languageState.tokens; + final RadixorBenchmarkStemmer stemmer = languageState.radixorStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Adds Lucene lower-case normalization to a measured filter pipeline. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream lowercase(final TokenStream input) { + return new LowerCaseFilter(input); + } + + /** + * Adds the German normalization path used by Lucene's German analyzer before + * German light/minimal stemming. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream germanNormalize(final TokenStream input) { + return new GermanNormalizationFilter(lowercase(input)); + } + + /** + * Adds the Persian normalization path used by Lucene's Persian analyzer before + * Persian stemming. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream persianNormalize(final TokenStream input) { + TokenStream result = lowercase(input); + result = new DecimalDigitFilter(result); + result = new ArabicNormalizationFilter(result); + result = new PersianNormalizationFilter(result); + return result; + } + + /** + * Loads the benchmark-only Ukrainian Morfologik dictionary. + * + * @return Ukrainian Morfologik dictionary + * @throws IOException if the dictionary cannot be loaded + */ + private static Dictionary loadUkrainianMorfologikDictionary() throws IOException { + final URL dictionaryUrl = MultiLanguageStemmerComparisonBenchmark.class.getClassLoader() + .getResource("ua/net/nlp/ukrainian.dict"); + if (dictionaryUrl == null) { + throw new IllegalStateException("Missing Ukrainian Morfologik dictionary resource."); + } + return Dictionary.read(dictionaryUrl); + } + + /** + * Returns the first Morfologik stem for one token. + * + * @param token input token + * @param lookup dictionary lookup + * @return first Morfologik stem, or the input token when no analysis exists + */ + private static String firstMorfologikStem(final String token, final DictionaryLookup lookup) { + final List analyses = lookup.lookup(token); + if (analyses.isEmpty()) { + return token; + } + return analyses.get(0).getStem().toString(); + } + + /** + * Immutable language-specific state. + */ + private static final class LanguageState { + + /** + * Shared deterministic changed-token dictionary corpus. + */ + private final String[] tokens; + + /** + * Radixor benchmark adapter. + */ + private final RadixorBenchmarkStemmer radixorStemmer; + + /** + * Creates language state. + * + * @param tokens token corpus + * @param radixorStemmer Radixor benchmark adapter + */ + private LanguageState(final String[] tokens, final RadixorBenchmarkStemmer radixorStemmer) { + this.tokens = tokens; + this.radixorStemmer = radixorStemmer; + } + } + + /** + * Factory for a Lucene filter under test. + */ + private interface FilterFactory { + + /** + * Creates a token stream wrapping the supplied benchmark input stream. + * + * @param input input token stream + * @return filter stream + */ + TokenStream create(TokenStream input); + } + + /** + * Reusable input stream, filter stream, and term attribute for one Lucene + * benchmark method. + */ + private static final class FilterPipeline { + + /** + * Reusable benchmark input stream. + */ + private final BenchmarkTokenStream input; + + /** + * Lucene filter output stream. + */ + private final TokenStream output; + + /** + * Term attribute consumed by the benchmark. + */ + private final CharTermAttribute termAttribute; + + /** + * Creates one reusable filter pipeline. + * + * @param factory filter factory + */ + private FilterPipeline(final FilterFactory factory) { + this.input = new BenchmarkTokenStream(new String[0]); + this.output = factory.create(this.input); + this.termAttribute = this.output.addAttribute(CharTermAttribute.class); + } + + /** + * Runs the filter over one token corpus and consumes all emitted terms. + * + *

    + * The benchmark intentionally rebinds the {@code String[]} corpus on every + * measured operation so the adaptation cost from the canonical string input to + * Lucene's mutable character attributes is included. + *

    + * + * @param tokens token corpus + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + private void run(final String[] tokens, final Blackhole blackhole) throws IOException { + this.input.setTokens(tokens); + this.output.reset(); + while (this.output.incrementToken()) { + blackhole.consume(this.termAttribute.toString()); + } + this.output.end(); + } + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java b/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java new file mode 100644 index 0000000..f63c4a7 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.util.Objects; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; + +/** + * Benchmark-only Radixor stemmer adapter for the canonical preferred-result + * path over normalized benchmark tokens. + * + *

    + * The benchmark corpus is normalized during setup, so this adapter uses + * {@link FrequencyTrie#getNormalizedString(String)} to avoid measuring + * redundant lookup-time normalization. Patch commands are applied with the + * traversal direction persisted in the trie metadata. + *

    + * + *

    + * Instances are mutable and intended for one JMH worker thread. + *

    + */ +final class RadixorBenchmarkStemmer { + + /** + * Compiled Radixor patch trie with decoded patch-command values. + */ + private final FrequencyTrie trie; + + /** + * Creates a benchmark stemmer around one compiled Radixor trie. + * + * @param trie compiled Radixor patch trie + */ + RadixorBenchmarkStemmer(final FrequencyTrie trie) { + this.trie = Objects.requireNonNull(trie, "trie"); + } + + /** + * Stems one benchmark token through the canonical trie lookup API. + * + * @param token input token + * @return Radixor stem or the input token when no patch is stored + */ + String stem(final String token) { + final CompiledPatchCommand patch = this.trie.getNormalizedString(token); + if (patch == null || patch.preservesAllSources()) { + return token; + } + return patch.apply(token); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java new file mode 100644 index 0000000..a015eda --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java @@ -0,0 +1,196 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.danishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.dutchStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.finnishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.frenchStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.germanStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.hungarianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.italianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.norwegianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.portugueseStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.russianStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.spanishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.swedishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.yiddishStemmer; + +/** + * Maps Radixor dictionary languages to matching official Snowball algorithms. + */ +enum SnowballLanguageCase { + + /** + * Danish Snowball stemming over the Radixor Danish dictionary. + */ + DANISH("Danish", StemmerPatchTrieLoader.Language.DA_DK, danishStemmer::new, "Danish"), + + /** + * Dutch Snowball stemming over the Radixor Dutch dictionary. + */ + DUTCH("Dutch", StemmerPatchTrieLoader.Language.NL_NL, dutchStemmer::new, "Dutch"), + + /** + * Finnish Snowball stemming over the Radixor Finnish dictionary. + */ + FINNISH("Finnish", StemmerPatchTrieLoader.Language.FI_FI, finnishStemmer::new, "Finnish"), + + /** + * French Snowball stemming over the Radixor French dictionary. + */ + FRENCH("French", StemmerPatchTrieLoader.Language.FR_FR, frenchStemmer::new, "French"), + + /** + * German Snowball stemming over the Radixor German dictionary. + */ + GERMAN("German", StemmerPatchTrieLoader.Language.DE_DE, germanStemmer::new, "German"), + + /** + * Hungarian Snowball stemming over the Radixor Hungarian dictionary. + */ + HUNGARIAN("Hungarian", StemmerPatchTrieLoader.Language.HU_HU, hungarianStemmer::new, "Hungarian"), + + /** + * Italian Snowball stemming over the Radixor Italian dictionary. + */ + ITALIAN("Italian", StemmerPatchTrieLoader.Language.IT_IT, italianStemmer::new, "Italian"), + + /** + * Norwegian Snowball stemming over the Radixor Bokmal dictionary. + */ + NORWEGIAN_BOKMAL("Norwegian Bokmal", StemmerPatchTrieLoader.Language.NB_NO, norwegianStemmer::new, + "Norwegian"), + + /** + * Norwegian Snowball stemming over the Radixor Nynorsk dictionary. + */ + NORWEGIAN_NYNORSK("Norwegian Nynorsk", StemmerPatchTrieLoader.Language.NN_NO, norwegianStemmer::new, + "Norwegian"), + + /** + * Portuguese Snowball stemming over the Radixor Portuguese dictionary. + */ + PORTUGUESE("Portuguese", StemmerPatchTrieLoader.Language.PT_PT, portugueseStemmer::new, "Portuguese"), + + /** + * Russian Snowball stemming over the Radixor Russian dictionary. + */ + RUSSIAN("Russian", StemmerPatchTrieLoader.Language.RU_RU, russianStemmer::new, "Russian"), + + /** + * Spanish Snowball stemming over the Radixor Spanish dictionary. + */ + SPANISH("Spanish", StemmerPatchTrieLoader.Language.ES_ES, spanishStemmer::new, "Spanish"), + + /** + * Swedish Snowball stemming over the Radixor Swedish dictionary. + */ + SWEDISH("Swedish", StemmerPatchTrieLoader.Language.SV_SE, swedishStemmer::new, "Swedish"), + + /** + * Yiddish Snowball stemming over the Radixor Yiddish dictionary. + */ + YIDDISH("Yiddish", StemmerPatchTrieLoader.Language.YI, yiddishStemmer::new, "Yiddish"); + + /** + * Human-readable language name. + */ + private final String displayLanguage; + + /** + * Matching Radixor language resource. + */ + private final StemmerPatchTrieLoader.Language radixorLanguage; + + /** + * Factory for the isolated benchmark-only Snowball implementation. + */ + private final SnowballStemmerAdapter.Factory directFactory; + + /** + * Lucene SnowballFilter algorithm name. + */ + private final String luceneSnowballName; + + /** + * Creates a language case. + * + * @param displayLanguage human-readable language name + * @param radixorLanguage matching Radixor language resource + * @param directFactory direct Snowball stemmer factory + * @param luceneSnowballName Lucene SnowballFilter algorithm name + */ + SnowballLanguageCase(final String displayLanguage, final StemmerPatchTrieLoader.Language radixorLanguage, + final SnowballStemmerAdapter.Factory directFactory, final String luceneSnowballName) { + this.displayLanguage = displayLanguage; + this.radixorLanguage = radixorLanguage; + this.directFactory = directFactory; + this.luceneSnowballName = luceneSnowballName; + } + + /** + * Returns the human-readable language name. + * + * @return display language + */ + String displayLanguage() { + return this.displayLanguage; + } + + /** + * Returns the matching Radixor dictionary language. + * + * @return Radixor language + */ + StemmerPatchTrieLoader.Language radixorLanguage() { + return this.radixorLanguage; + } + + /** + * Creates a direct Snowball stemmer adapter. + * + * @return direct Snowball adapter + */ + SnowballStemmerAdapter createDirectStemmer() { + return new SnowballStemmerAdapter(this.directFactory); + } + + /** + * Returns the Lucene SnowballFilter algorithm name. + * + * @return Lucene SnowballFilter algorithm name + */ + String luceneSnowballName() { + return this.luceneSnowballName; + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java new file mode 100644 index 0000000..4d4bf01 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java @@ -0,0 +1,246 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.snowball.SnowballFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Compares Radixor with official Snowball algorithms for every Radixor language + * that has a matching Snowball Java stemmer. + * + *

    + * Each benchmark operation processes the same changed-token Radixor + * dictionary-derived language corpus, repeated only when the changed-token + * resource contains fewer than 5,000 token fields. The direct Snowball method + * measures the isolated benchmark-only Snowball source. The Lucene + * SnowballFilter method measures Lucene's TokenStream integration path, + * including lower-case normalization and token attribute overhead. + *

    + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class SnowballLanguageStemmerComparisonBenchmark { + + /** + * Shared language corpus and Radixor trie state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Language/algorithm case under comparison. + */ + @Param({ "DANISH", "DUTCH", "FINNISH", "FRENCH", "GERMAN", "HUNGARIAN", "ITALIAN", + "NORWEGIAN_BOKMAL", "NORWEGIAN_NYNORSK", "PORTUGUESE", "RUSSIAN", "SPANISH", "SWEDISH", + "YIDDISH" }) + public String languageCaseName; + + /** + * Resolved language/algorithm case. + */ + private SnowballLanguageCase languageCase; + + /** + * Shared deterministic changed-token dictionary corpus. + */ + private String[] tokens; + + /** + * Compiled Radixor trie for the selected language. + */ + private RadixorBenchmarkStemmer radixorStemmer; + + /** + * Initializes shared language resources before measurement. + * + * @throws IOException if the corpus or trie cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.languageCase = SnowballLanguageCase.valueOf(this.languageCaseName); + this.tokens = LanguageBenchmarkCorpus.createTokens(this.languageCase.radixorLanguage()); + this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + this.languageCase.radixorLanguage(), true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + } + } + + /** + * Per-thread direct Snowball state. + */ + @State(Scope.Thread) + public static class DirectState { + + /** + * Reusable direct Snowball adapter. + */ + private SnowballStemmerAdapter snowballStemmer; + + /** + * Initializes direct Snowball state for the selected language. + * + * @param sharedState selected language state + */ + @Setup(Level.Trial) + public void setUp(final SharedState sharedState) { + this.snowballStemmer = sharedState.languageCase.createDirectStemmer(); + } + } + + /** + * Per-thread Lucene SnowballFilter state. + */ + @State(Scope.Thread) + public static class LuceneSnowballState { + + /** + * Reusable benchmark input stream. + */ + private BenchmarkTokenStream input; + + /** + * Reusable Lucene SnowballFilter output stream. + */ + private TokenStream output; + + /** + * Reusable term attribute. + */ + private CharTermAttribute termAttribute; + + /** + * Initializes Lucene SnowballFilter state for the selected language. + * + * @param sharedState selected language state + */ + @Setup(Level.Trial) + public void setUp(final SharedState sharedState) { + this.input = new BenchmarkTokenStream(new String[0]); + final TokenStream normalizedInput = new LowerCaseFilter(this.input); + this.output = new SnowballFilter(normalizedInput, sharedState.languageCase.luceneSnowballName()); + this.termAttribute = this.output.addAttribute(CharTermAttribute.class); + } + + /** + * Runs the reusable Lucene SnowballFilter over one corpus. + * + *

    + * The benchmark intentionally rebinds the {@code String[]} corpus on every + * measured operation so the adaptation cost from the canonical string input to + * Lucene's mutable character attributes is included. + *

    + * + * @param tokens token corpus + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + void run(final String[] tokens, final Blackhole blackhole) throws IOException { + this.input.setTokens(tokens); + this.output.reset(); + while (this.output.incrementToken()) { + blackhole.consume(this.termAttribute.toString()); + } + this.output.end(); + } + } + + /** + * Runs Radixor over the selected Snowball-language corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void radixor(final SharedState sharedState, final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Runs the official Snowball direct Java implementation over the selected + * language corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct Snowball state + * @param blackhole result sink + */ + @Benchmark + public void snowballDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final SnowballStemmerAdapter stemmer = directState.snowballStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Runs Lucene SnowballFilter over the selected language corpus. + * + * @param sharedState shared benchmark state + * @param luceneState reusable Lucene Snowball state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void luceneSnowballFilter(final SharedState sharedState, final LuceneSnowballState luceneState, + final Blackhole blackhole) throws IOException { + luceneState.run(sharedState.tokens, blackhole); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java index 12bd12a..0044e03 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java @@ -32,7 +32,7 @@ package org.egothor.stemmer.benchmark; import java.util.Objects; -import org.tartarus.snowball.SnowballStemmer; +import org.egothor.stemmer.benchmark.snowball.SnowballStemmer; /** * Small adapter around a Snowball stemmer instance used by benchmarks. diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java new file mode 100644 index 0000000..f9978db --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java @@ -0,0 +1,813 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.ar.ArabicNormalizationFilter; +import org.apache.lucene.analysis.core.DecimalDigitFilter; +import org.apache.lucene.analysis.cz.CzechStemFilter; +import org.apache.lucene.analysis.de.GermanLightStemFilter; +import org.apache.lucene.analysis.de.GermanMinimalStemFilter; +import org.apache.lucene.analysis.de.GermanNormalizationFilter; +import org.apache.lucene.analysis.de.GermanStemFilter; +import org.apache.lucene.analysis.en.EnglishMinimalStemFilter; +import org.apache.lucene.analysis.en.EnglishPossessiveFilter; +import org.apache.lucene.analysis.en.KStemFilter; +import org.apache.lucene.analysis.en.PorterStemFilter; +import org.apache.lucene.analysis.es.SpanishLightStemFilter; +import org.apache.lucene.analysis.es.SpanishMinimalStemFilter; +import org.apache.lucene.analysis.es.SpanishPluralStemFilter; +import org.apache.lucene.analysis.fa.PersianNormalizationFilter; +import org.apache.lucene.analysis.fa.PersianStemFilter; +import org.apache.lucene.analysis.fi.FinnishLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter; +import org.apache.lucene.analysis.hu.HungarianLightStemFilter; +import org.apache.lucene.analysis.it.ItalianLightStemFilter; +import org.apache.lucene.analysis.morfologik.MorfologikFilter; +import org.apache.lucene.analysis.no.NorwegianLightStemFilter; +import org.apache.lucene.analysis.no.NorwegianMinimalStemFilter; +import org.apache.lucene.analysis.pl.PolishAnalyzer; +import org.apache.lucene.analysis.pt.PortugueseLightStemFilter; +import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter; +import org.apache.lucene.analysis.pt.PortugueseStemFilter; +import org.apache.lucene.analysis.ru.RussianLightStemFilter; +import org.apache.lucene.analysis.snowball.SnowballFilter; +import org.apache.lucene.analysis.stempel.StempelFilter; +import org.apache.lucene.analysis.stempel.StempelStemmer; +import org.apache.lucene.analysis.sv.SwedishLightStemFilter; +import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.englishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import morfologik.stemming.Dictionary; +import morfologik.stemming.DictionaryLookup; +import morfologik.stemming.WordData; + +/** + * Emits exact-root agreement metrics through standard JMH result files. + * + *

    + * This benchmark is a quality pass, not a throughput competitor. Each operation + * evaluates one stemmer against the complete Radixor dictionary resource for + * the matching language. The useful outputs are the JMH auxiliary counters + * {@code correctMatches}, {@code evaluatedTokens}, + * {@code changedCorrectMatches}, {@code changedEvaluatedTokens}, + * {@code rootPreservedMatches}, and {@code rootEvaluatedTokens}. + *

    + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class StemmerComparisonBenchmarkQuality { + + /** + * Shared quality state for one candidate stemmer. + */ + @State(Scope.Benchmark) + public static class QualityState { + + /** + * Candidate stemmer whose exact-root agreement is measured. + */ + @Param({ + "ENGLISH_RADIXOR", + "ENGLISH_SNOWBALL_ORIGINAL_PORTER", + "ENGLISH_SNOWBALL_PORTER2", + "ENGLISH_LUCENE_PORTER_COPIED", + "ENGLISH_LUCENE_PORTER_FILTER", + "ENGLISH_LUCENE_KSTEM_FILTER", + "ENGLISH_LUCENE_MINIMAL_FILTER", + "ENGLISH_LUCENE_POSSESSIVE_FILTER", + "ENGLISH_PAICE_HUSK_LANCASTER", + "ENGLISH_OPENNLP_PORTER", + "CZECH_RADIXOR", + "CZECH_LUCENE_CZECH_STEM_FILTER", + "GERMAN_RADIXOR", + "GERMAN_LUCENE_GERMAN_STEM_FILTER", + "GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER", + "GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER", + "SPANISH_RADIXOR", + "SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER", + "SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER", + "SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER", + "PERSIAN_RADIXOR", + "PERSIAN_LUCENE_PERSIAN_STEM_FILTER", + "FINNISH_RADIXOR", + "FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER", + "FRENCH_RADIXOR", + "FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER", + "FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER", + "HUNGARIAN_RADIXOR", + "HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER", + "ITALIAN_RADIXOR", + "ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER", + "NORWEGIAN_BOKMAL_RADIXOR", + "NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER", + "NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER", + "POLISH_RADIXOR", + "POLISH_LUCENE_STEMPEL_DIRECT", + "POLISH_LUCENE_STEMPEL_FILTER", + "POLISH_LUCENE_MORFOLOGIK_FILTER", + "PORTUGUESE_RADIXOR", + "PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER", + "PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER", + "PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER", + "RUSSIAN_RADIXOR", + "RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER", + "SWEDISH_RADIXOR", + "SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER", + "SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER", + "UKRAINIAN_RADIXOR", + "UKRAINIAN_MORFOLOGIK_DIRECT", + "UKRAINIAN_LUCENE_MORFOLOGIK_FILTER", + "SNOWBALL_DANISH_RADIXOR", + "SNOWBALL_DANISH_DIRECT", + "SNOWBALL_DANISH_LUCENE_FILTER", + "SNOWBALL_DUTCH_RADIXOR", + "SNOWBALL_DUTCH_DIRECT", + "SNOWBALL_DUTCH_LUCENE_FILTER", + "SNOWBALL_FINNISH_RADIXOR", + "SNOWBALL_FINNISH_DIRECT", + "SNOWBALL_FINNISH_LUCENE_FILTER", + "SNOWBALL_FRENCH_RADIXOR", + "SNOWBALL_FRENCH_DIRECT", + "SNOWBALL_FRENCH_LUCENE_FILTER", + "SNOWBALL_GERMAN_RADIXOR", + "SNOWBALL_GERMAN_DIRECT", + "SNOWBALL_GERMAN_LUCENE_FILTER", + "SNOWBALL_HUNGARIAN_RADIXOR", + "SNOWBALL_HUNGARIAN_DIRECT", + "SNOWBALL_HUNGARIAN_LUCENE_FILTER", + "SNOWBALL_ITALIAN_RADIXOR", + "SNOWBALL_ITALIAN_DIRECT", + "SNOWBALL_ITALIAN_LUCENE_FILTER", + "SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR", + "SNOWBALL_NORWEGIAN_BOKMAL_DIRECT", + "SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER", + "SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR", + "SNOWBALL_NORWEGIAN_NYNORSK_DIRECT", + "SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER", + "SNOWBALL_PORTUGUESE_RADIXOR", + "SNOWBALL_PORTUGUESE_DIRECT", + "SNOWBALL_PORTUGUESE_LUCENE_FILTER", + "SNOWBALL_RUSSIAN_RADIXOR", + "SNOWBALL_RUSSIAN_DIRECT", + "SNOWBALL_RUSSIAN_LUCENE_FILTER", + "SNOWBALL_SPANISH_RADIXOR", + "SNOWBALL_SPANISH_DIRECT", + "SNOWBALL_SPANISH_LUCENE_FILTER", + "SNOWBALL_SWEDISH_RADIXOR", + "SNOWBALL_SWEDISH_DIRECT", + "SNOWBALL_SWEDISH_LUCENE_FILTER", + "SNOWBALL_YIDDISH_RADIXOR", + "SNOWBALL_YIDDISH_DIRECT", + "SNOWBALL_YIDDISH_LUCENE_FILTER" + }) + public String candidateName; + + /** + * Full dictionary corpus for the selected language. + */ + private LanguageBenchmarkCorpus.Corpus corpus; + + /** + * Candidate evaluator. + */ + private QualityEvaluator evaluator; + + /** + * Initializes corpus and evaluator before measurement. + * + * @throws IOException if dictionary or stemmer resources cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final QualityCandidate candidate = QualityCandidate.valueOf(this.candidateName); + this.corpus = LanguageBenchmarkCorpus.createFullCorpus(candidate.radixorLanguage()); + this.evaluator = candidate.createEvaluator(); + } + } + + /** + * JMH auxiliary counters for exact-root agreement. + */ + @State(Scope.Thread) + @AuxCounters(AuxCounters.Type.EVENTS) + public static class QualityCounters { + + /** + * Number of outputs equal to the dictionary root. + */ + public long correctMatches; + + /** + * Number of evaluated input tokens. + */ + public long evaluatedTokens; + + /** + * Number of exact-root matches where the input token differs from the + * expected root. + */ + public long changedCorrectMatches; + + /** + * Number of evaluated tokens where the input token differs from the expected + * root. + */ + public long changedEvaluatedTokens; + + /** + * Number of exact-root matches where the input token is already the expected + * root. + */ + public long rootPreservedMatches; + + /** + * Number of evaluated tokens where the input token is already the expected + * root. + */ + public long rootEvaluatedTokens; + + /** + * Resets counters before each measured iteration. + */ + @Setup(Level.Iteration) + public void reset() { + this.correctMatches = 0L; + this.evaluatedTokens = 0L; + this.changedCorrectMatches = 0L; + this.changedEvaluatedTokens = 0L; + this.rootPreservedMatches = 0L; + this.rootEvaluatedTokens = 0L; + } + } + + /** + * Runs exact-root agreement over the full dictionary corpus. + * + * @param state quality state + * @param counters auxiliary JMH counters + * @param blackhole result sink + * @return exact-root match count for this operation + * @throws IOException if Lucene streaming fails + */ + @Benchmark + public int exactRootAgreement(final QualityState state, final QualityCounters counters, final Blackhole blackhole) + throws IOException { + final QualityResult result = state.evaluator.evaluate(state.corpus, blackhole); + counters.correctMatches += result.correctMatches(); + counters.evaluatedTokens += result.evaluatedTokens(); + counters.changedCorrectMatches += result.changedCorrectMatches(); + counters.changedEvaluatedTokens += result.changedEvaluatedTokens(); + counters.rootPreservedMatches += result.rootPreservedMatches(); + counters.rootEvaluatedTokens += result.rootEvaluatedTokens(); + return result.correctMatches(); + } + + /** + * Candidate stemmers that can be evaluated against a Radixor resource. + */ + private enum QualityCandidate { + ENGLISH_RADIXOR(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_SNOWBALL_ORIGINAL_PORTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_SNOWBALL_PORTER2(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_PORTER_COPIED(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_PORTER_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_KSTEM_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_MINIMAL_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_POSSESSIVE_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_PAICE_HUSK_LANCASTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_OPENNLP_PORTER(StemmerPatchTrieLoader.Language.US_UK), + CZECH_RADIXOR(StemmerPatchTrieLoader.Language.CS_CZ), + CZECH_LUCENE_CZECH_STEM_FILTER(StemmerPatchTrieLoader.Language.CS_CZ), + GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + PERSIAN_RADIXOR(StemmerPatchTrieLoader.Language.FA_IR), + PERSIAN_LUCENE_PERSIAN_STEM_FILTER(StemmerPatchTrieLoader.Language.FA_IR), + FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI), + FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FI_FI), + FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR), + FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR), + FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR), + HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU), + HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.HU_HU), + ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT), + ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.IT_IT), + NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO), + NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO), + NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO), + POLISH_RADIXOR(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_STEMPEL_DIRECT(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_STEMPEL_FILTER(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.PL_PL), + PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU), + RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.RU_RU), + SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE), + SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE), + SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE), + UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA), + UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA), + UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA), + SNOWBALL_DANISH_RADIXOR(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DUTCH_RADIXOR(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_DUTCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FINNISH_DIRECT(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FINNISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_FRENCH_DIRECT(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_FRENCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_GERMAN_DIRECT(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_GERMAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_HUNGARIAN_DIRECT(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_HUNGARIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_ITALIAN_DIRECT(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_ITALIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_BOKMAL_DIRECT(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_NORWEGIAN_NYNORSK_DIRECT(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_RUSSIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SPANISH_DIRECT(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SPANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_SWEDISH_DIRECT(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_SWEDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_YIDDISH_RADIXOR(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH), + SNOWBALL_YIDDISH_DIRECT(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH), + SNOWBALL_YIDDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH); + + /** + * Radixor dictionary language used as truth. + */ + private final StemmerPatchTrieLoader.Language radixorLanguage; + + /** + * Optional Snowball language mapping. + */ + private final SnowballLanguageCase snowballLanguageCase; + + /** + * Creates a candidate. + * + * @param radixorLanguage Radixor dictionary language + */ + QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage) { + this(radixorLanguage, null); + } + + /** + * Creates a candidate. + * + * @param radixorLanguage Radixor dictionary language + * @param snowballLanguageCase matching Snowball case + */ + QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage, + final SnowballLanguageCase snowballLanguageCase) { + this.radixorLanguage = radixorLanguage; + this.snowballLanguageCase = snowballLanguageCase; + } + + /** + * Returns the Radixor dictionary language. + * + * @return Radixor language + */ + StemmerPatchTrieLoader.Language radixorLanguage() { + return this.radixorLanguage; + } + + /** + * Creates the evaluator for this candidate. + * + * @return quality evaluator + * @throws IOException if stemmer resources cannot be loaded + */ + QualityEvaluator createEvaluator() throws IOException { + if (name().endsWith("_RADIXOR")) { + return direct(createRadixorStemmer(this.radixorLanguage)); + } + if (name().endsWith("_DIRECT") && this.snowballLanguageCase != null) { + return direct(this.snowballLanguageCase.createDirectStemmer()::stem); + } + if (name().endsWith("_LUCENE_FILTER") && this.snowballLanguageCase != null) { + return tokenFilter(input -> new SnowballFilter(new LowerCaseFilter(input), + this.snowballLanguageCase.luceneSnowballName())); + } + + return switch (this) { + case ENGLISH_SNOWBALL_ORIGINAL_PORTER -> direct(new SnowballStemmerAdapter(porterStemmer::new)::stem); + case ENGLISH_SNOWBALL_PORTER2 -> direct(new SnowballStemmerAdapter(englishStemmer::new)::stem); + case ENGLISH_LUCENE_PORTER_COPIED -> direct(new LucenePorterStemmerCopied()::stem); + case ENGLISH_LUCENE_PORTER_FILTER -> tokenFilter(PorterStemFilter::new); + case ENGLISH_LUCENE_KSTEM_FILTER -> tokenFilter(KStemFilter::new); + case ENGLISH_LUCENE_MINIMAL_FILTER -> tokenFilter(EnglishMinimalStemFilter::new); + case ENGLISH_LUCENE_POSSESSIVE_FILTER -> tokenFilter(EnglishPossessiveFilter::new); + case ENGLISH_PAICE_HUSK_LANCASTER -> direct(new PaiceHuskLancasterStemmer()::stem); + case ENGLISH_OPENNLP_PORTER -> { + final opennlp.tools.stemmer.PorterStemmer stemmer = + new opennlp.tools.stemmer.PorterStemmer(); + yield direct(token -> stemmer.stem(token).toString()); + } + case CZECH_LUCENE_CZECH_STEM_FILTER -> tokenFilter(input -> new CzechStemFilter(lowercase(input))); + case GERMAN_LUCENE_GERMAN_STEM_FILTER -> tokenFilter(input -> new GermanStemFilter(lowercase(input))); + case GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new GermanLightStemFilter(germanNormalize(input))); + case GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new GermanMinimalStemFilter(germanNormalize(input))); + case SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new SpanishLightStemFilter(lowercase(input))); + case SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new SpanishMinimalStemFilter(lowercase(input))); + case SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER -> + tokenFilter(input -> new SpanishPluralStemFilter(lowercase(input))); + case PERSIAN_LUCENE_PERSIAN_STEM_FILTER -> + tokenFilter(input -> new PersianStemFilter(persianNormalize(input))); + case FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new FinnishLightStemFilter(lowercase(input))); + case FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new FrenchLightStemFilter(lowercase(input))); + case FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new FrenchMinimalStemFilter(lowercase(input))); + case HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new HungarianLightStemFilter(lowercase(input))); + case ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new ItalianLightStemFilter(lowercase(input))); + case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new NorwegianLightStemFilter(lowercase(input))); + case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new NorwegianMinimalStemFilter(lowercase(input))); + case POLISH_LUCENE_STEMPEL_DIRECT -> { + final StempelStemmer stemmer = new StempelStemmer(PolishAnalyzer.getDefaultTable()); + yield direct(token -> { + final StringBuilder stem = stemmer.stem(token); + return stem == null ? token : stem.toString(); + }); + } + case POLISH_LUCENE_STEMPEL_FILTER -> + tokenFilter(input -> new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()))); + case POLISH_LUCENE_MORFOLOGIK_FILTER -> tokenFilter(MorfologikFilter::new); + case PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER -> + tokenFilter(input -> new PortugueseStemFilter(lowercase(input))); + case PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER -> + tokenFilter(input -> new PortugueseLightStemFilter(lowercase(input))); + case PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new PortugueseMinimalStemFilter(lowercase(input))); + case RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new RussianLightStemFilter(lowercase(input))); + case SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new SwedishLightStemFilter(lowercase(input))); + case SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new SwedishMinimalStemFilter(lowercase(input))); + case UKRAINIAN_MORFOLOGIK_DIRECT -> { + final DictionaryLookup lookup = new DictionaryLookup(loadUkrainianMorfologikDictionary()); + yield direct(token -> firstMorfologikStem(token, lookup)); + } + case UKRAINIAN_LUCENE_MORFOLOGIK_FILTER -> { + final Dictionary dictionary = loadUkrainianMorfologikDictionary(); + yield tokenFilter(input -> new MorfologikFilter(input, dictionary)); + } + default -> throw new IllegalStateException("No evaluator for " + this + "."); + }; + } + } + + /** + * Direct stemmer function. + */ + @FunctionalInterface + private interface Stemmer { + + /** + * Produces one stem. + * + * @param token input token + * @return produced stem + */ + String stem(String token); + } + + /** + * Quality evaluator for one candidate. + */ + @FunctionalInterface + private interface QualityEvaluator { + + /** + * Evaluates exact-root agreement for one corpus. + * + * @param corpus token/root corpus + * @param blackhole result sink + * @return exact-root match count + * @throws IOException if Lucene streaming fails + */ + QualityResult evaluate(LanguageBenchmarkCorpus.Corpus corpus, Blackhole blackhole) throws IOException; + } + + /** + * Exact-root agreement counters for one quality operation. + * + * @param correctMatches total exact-root matches + * @param evaluatedTokens total evaluated tokens + * @param changedCorrectMatches exact-root matches where token differs from root + * @param changedEvaluatedTokens evaluated tokens where token differs from root + * @param rootPreservedMatches exact-root matches where token already equals root + * @param rootEvaluatedTokens evaluated tokens where token already equals root + */ + private record QualityResult(int correctMatches, int evaluatedTokens, int changedCorrectMatches, + int changedEvaluatedTokens, int rootPreservedMatches, int rootEvaluatedTokens) { + } + + /** + * Creates a direct evaluator. + * + * @param stemmer direct stemmer + * @return quality evaluator + */ + private static QualityEvaluator direct(final Stemmer stemmer) { + Objects.requireNonNull(stemmer, "stemmer"); + return (corpus, blackhole) -> { + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + final String[] tokens = corpus.tokens(); + final String[] expectedRoots = corpus.expectedRoots(); + for (int index = 0; index < tokens.length; index++) { + final String token = tokens[index]; + final String expectedRoot = expectedRoots[index]; + final String actual = stemmer.stem(token); + blackhole.consume(actual); + final boolean exact = Objects.equals(expectedRoot, actual); + if (exact) { + correct++; + } + if (Objects.equals(token, expectedRoot)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + return new QualityResult(correct, tokens.length, changedCorrect, changedEvaluated, rootPreserved, + rootEvaluated); + }; + } + + /** + * Creates a TokenFilter evaluator. + * + * @param factory token stream factory + * @return quality evaluator + */ + private static QualityEvaluator tokenFilter(final Function factory) { + Objects.requireNonNull(factory, "factory"); + return (corpus, blackhole) -> { + final String[] actualStems = firstTokenFilterOutputs(corpus.tokens(), factory, blackhole); + final String[] expectedRoots = corpus.expectedRoots(); + final String[] tokens = corpus.tokens(); + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + for (int index = 0; index < actualStems.length; index++) { + final String token = tokens[index]; + final String expectedRoot = expectedRoots[index]; + final boolean exact = Objects.equals(expectedRoot, actualStems[index]); + if (exact) { + correct++; + } + if (Objects.equals(token, expectedRoot)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + return new QualityResult(correct, actualStems.length, changedCorrect, changedEvaluated, rootPreserved, + rootEvaluated); + }; + } + + /** + * Creates a direct Radixor stemmer. + * + * @param language Radixor dictionary language + * @return direct stemmer + * @throws IOException if the trie cannot be loaded + */ + private static Stemmer createRadixorStemmer(final StemmerPatchTrieLoader.Language language) throws IOException { + final RadixorBenchmarkStemmer stemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + language, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + return stemmer::stem; + } + + /** + * Loads the benchmark-only Ukrainian Morfologik dictionary. + * + * @return Ukrainian Morfologik dictionary + * @throws IOException if the dictionary cannot be loaded + */ + private static Dictionary loadUkrainianMorfologikDictionary() throws IOException { + final URL dictionaryUrl = StemmerComparisonBenchmarkQuality.class.getClassLoader() + .getResource("ua/net/nlp/ukrainian.dict"); + if (dictionaryUrl == null) { + throw new IllegalStateException("Missing Ukrainian Morfologik dictionary resource."); + } + return Dictionary.read(dictionaryUrl); + } + + /** + * Returns the first Morfologik stem for one token. + * + * @param token input token + * @param lookup dictionary lookup + * @return first Morfologik stem, or the input token when no analysis exists + */ + private static String firstMorfologikStem(final String token, final DictionaryLookup lookup) { + final List analyses = lookup.lookup(token); + if (analyses.isEmpty()) { + return token; + } + return analyses.get(0).getStem().toString(); + } + + /** + * Extracts the first emitted term for each input token from a TokenFilter + * pipeline. + * + * @param tokens token corpus + * @param factory token stream factory + * @param blackhole result sink + * @return first emitted term per input token + * @throws IOException if Lucene streaming fails + */ + private static String[] firstTokenFilterOutputs(final String[] tokens, final Function factory, + final Blackhole blackhole) throws IOException { + final String[] outputs = new String[tokens.length]; + final BenchmarkTokenStream input = new BenchmarkTokenStream(tokens); + final TokenStream output = factory.apply(input); + final CharTermAttribute termAttribute = output.addAttribute(CharTermAttribute.class); + final PositionIncrementAttribute positionAttribute = output.addAttribute(PositionIncrementAttribute.class); + int inputIndex = -1; + boolean recordedForPosition = false; + + output.reset(); + while (output.incrementToken()) { + final int positionIncrement = positionAttribute.getPositionIncrement(); + if (positionIncrement > 0) { + inputIndex += positionIncrement; + recordedForPosition = false; + } + if (inputIndex >= 0 && inputIndex < outputs.length && !recordedForPosition) { + outputs[inputIndex] = termAttribute.toString(); + recordedForPosition = true; + } + blackhole.consume(termAttribute); + } + output.end(); + output.close(); + + for (int index = 0; index < outputs.length; index++) { + if (outputs[index] == null) { + outputs[index] = tokens[index]; + } + } + return outputs; + } + + /** + * Adds Lucene lower-case normalization. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream lowercase(final TokenStream input) { + return new LowerCaseFilter(input); + } + + /** + * Adds Lucene German normalization. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream germanNormalize(final TokenStream input) { + return new GermanNormalizationFilter(lowercase(input)); + } + + /** + * Adds Lucene Persian normalization. + * + * @param input input token stream + * @return normalized stream + */ + private static TokenStream persianNormalize(final TokenStream input) { + TokenStream result = lowercase(input); + result = new DecimalDigitFilter(result); + result = new ArabicNormalizationFilter(result); + result = new PersianNormalizationFilter(result); + return result; + } +} diff --git a/src/main/java/org/egothor/stemmer/Compile.java b/src/main/java/org/egothor/stemmer/Compile.java index 9a8c9fe..6300113 100644 --- a/src/main/java/org/egothor/stemmer/Compile.java +++ b/src/main/java/org/egothor/stemmer/Compile.java @@ -147,6 +147,7 @@ public final class Compile { * @param arguments parsed command-line arguments * @throws IOException if compilation or output writing fails */ + @SuppressWarnings("deprecation") private static void compile(final Arguments arguments) throws IOException { final ReductionSettings reductionSettings = new ReductionSettings(arguments.reductionMode(), arguments.dominantWinnerMinPercent(), arguments.dominantWinnerOverSecondRatio()); diff --git a/src/main/java/org/egothor/stemmer/CompiledPatchCommand.java b/src/main/java/org/egothor/stemmer/CompiledPatchCommand.java new file mode 100644 index 0000000..82454be --- /dev/null +++ b/src/main/java/org/egothor/stemmer/CompiledPatchCommand.java @@ -0,0 +1,1282 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer; + +import java.util.Objects; + +/** + * Immutable runtime command produced from one serialized Radixor patch command. + * + *

    + * Compilation selects a concrete command class for the patch shape. Common + * one-operation commands such as suffix deletion, prefix deletion, character + * append, character prepend, and single-character replacement therefore execute + * without a per-application opcode switch. Multi-operation patches are represented + * as a compound command containing concrete atomic operations. + *

    + * + *

    + * Instances are immutable and thread-safe. Setup code may cache and share them + * freely across tries and benchmark states. + *

    + */ +@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.UseVarargs" }) +public abstract class CompiledPatchCommand { + + /** + * Return value used when the caller-owned output range is too small. + */ + public static final int APPLY_INSUFFICIENT_CAPACITY = PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY; + + /** + * Opcode for deleting one or more characters. + */ + private static final char DELETE_OPCODE = 'D'; + + /** + * Opcode for inserting one character. + */ + private static final char INSERT_OPCODE = 'I'; + + /** + * Opcode for replacing one character. + */ + private static final char REPLACE_OPCODE = 'R'; + + /** + * Opcode for skipping one or more unchanged characters. + */ + private static final char SKIP_OPCODE = '-'; + + /** + * Opcode for a canonical no-operation patch. + */ + private static final char NOOP_OPCODE = 'N'; + + /** + * Canonical no-operation patch argument. + */ + private static final char NOOP_ARGUMENT = 'a'; + + /** + * Serialized length of one opcode/argument patch command. + */ + private static final int SINGLE_COMMAND_LENGTH = 2; + + /** + * Smallest decoded skip/delete count accepted by the patch format. + */ + private static final int MINIMUM_COUNT = 1; + + /** + * First encoded count argument. + */ + private static final char FIRST_COUNT_ARGUMENT = 'a'; + + /** + * Prefix used in unsupported NOOP patch argument exceptions. + */ + private static final String MSG_NOOP = "Unsupported NOOP patch argument: "; + + /** + * Prefix used in unsupported patch opcode exceptions. + */ + private static final String MSG_OPCODE = "Unsupported patch opcode: "; + + /** + * Traversal direction used by this command. + */ + private final WordTraversalDirection traversalDirection; + + /** + * Constant result-length delta applied by this command. + */ + private final int lengthDelta; + + /** + * Minimum source length required before this command can be applied. + */ + private final int minimumSourceLength; + + /** + * Creates one compiled command. + * + * @param traversalDirection traversal direction used by this command + * @param lengthDelta constant result-length delta for this command + * @param minimumSourceLength minimum source length required for application + */ + protected CompiledPatchCommand(final WordTraversalDirection traversalDirection, final int lengthDelta, + final int minimumSourceLength) { + this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection"); + this.lengthDelta = lengthDelta; + this.minimumSourceLength = minimumSourceLength; + } + + /** + * Creates a builder that compiles one serialized patch command. + * + * @param patchCommand serialized patch command, or {@code null} for a + * preserve-only command + * @param traversalDirection traversal direction used by the patch command + * @return builder configured for the supplied command + * @throws NullPointerException if {@code traversalDirection} is {@code null} + */ + public static Builder builder(final String patchCommand, final WordTraversalDirection traversalDirection) { + return new Builder(patchCommand, traversalDirection); + } + + /** + * Compiles a serialized patch command for repeated application. + * + * @param patchCommand serialized patch command, or {@code null} for a + * preserve-only command + * @param traversalDirection traversal direction used by the patch command + * @return immutable compiled patch command + * @throws NullPointerException if {@code traversalDirection} is + * {@code null} + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public static CompiledPatchCommand compile(final String patchCommand, + final WordTraversalDirection traversalDirection) { + return builder(patchCommand, traversalDirection).build(); + } + + /** + * Applies this command to one source word and returns the transformed word. + * + * @param source source word + * @return transformed word, or {@code null} when {@code source} is + * {@code null} + */ + public final String apply(final String source) { + if (source == null) { + return null; + } + return applyNonNull(source); + } + + /** + * Applies this command from a character sequence into caller-owned output + * storage. + * + * @param source source text + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if the output range is invalid + */ + public final int applyTo(final CharSequence source, final char[] output, final int outputOffset, + final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + return applyTo(source, 0, source.length(), output, outputOffset, outputLength); + } + + /** + * Applies this command from a character-sequence slice into caller-owned output + * storage. + * + * @param source source text + * @param sourceOffset first source offset + * @param sourceLength number of source characters + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if any range is invalid + */ + public final int applyTo(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length()); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + + final int producedLength = computeAppliedLength(sourceLength); + if (producedLength > outputLength) { + return APPLY_INSUFFICIENT_CAPACITY; + } + applySequenceToOutput(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + return producedLength; + } + + /** + * Applies this command from a character-array slice into caller-owned output + * storage. + * + * @param source source storage + * @param sourceOffset first source offset + * @param sourceLength number of source characters + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if any range is invalid + * @throws IllegalArgumentException if source and output ranges overlap in the + * same array + */ + public final int applyTo(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength); + + final int producedLength = computeAppliedLength(sourceLength); + if (producedLength > outputLength) { + return APPLY_INSUFFICIENT_CAPACITY; + } + applyArrayToOutput(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + return producedLength; + } + + /** + * Returns this command traversal direction. + * + * @return traversal direction + */ + public final WordTraversalDirection traversalDirection() { + return this.traversalDirection; + } + + /** + * Reports whether this command preserves every non-null source unchanged. + * + *

    + * Hot paths can use this method to avoid output-buffer copying and result-string + * allocation for canonical no-operation dictionary entries. + *

    + * + * @return {@code true} when {@link #apply(String)} always returns the supplied + * source reference for non-null input + */ + public abstract boolean preservesAllSources(); + + /** + * Applies this command to a non-null source string. + * + * @param source non-null source word + * @return transformed word + */ + protected abstract String applyNonNull(String source); + + /** + * Computes the output length for a source of the supplied length. + * + * @param sourceLength source length + * @return output length + */ + protected final int computeAppliedLength(final int sourceLength) { + if (sourceLength < this.minimumSourceLength) { + return sourceLength; + } + final int appliedLength = sourceLength + this.lengthDelta; + return appliedLength < MINIMUM_COUNT ? sourceLength : appliedLength; + } + + /** + * Returns whether this command can produce a non-empty result for the supplied + * source length. + * + * @param sourceLength source length + * @return {@code true} when the constant command delta keeps the result + * non-empty + */ + protected final boolean hasApplicableLength(final int sourceLength) { + return sourceLength >= this.minimumSourceLength && sourceLength + this.lengthDelta >= MINIMUM_COUNT; + } + + /** + * Applies this command from a character sequence into caller-owned output. + * + * @param source source text + * @param sourceOffset first source offset + * @param sourceLength source length + * @param output output storage + * @param outputOffset first output offset + * @param producedLength computed produced length + */ + protected abstract void applySequenceToOutput(CharSequence source, int sourceOffset, int sourceLength, + char[] output, int outputOffset, int producedLength); + + /** + * Applies this command from a character array into caller-owned output. + * + * @param source source storage + * @param sourceOffset first source offset + * @param sourceLength source length + * @param output output storage + * @param outputOffset first output offset + * @param producedLength computed produced length + */ + protected abstract void applyArrayToOutput(char[] source, int sourceOffset, int sourceLength, + char[] output, int outputOffset, int producedLength); + + /** + * Builder that compiles one serialized patch command to the most specific + * runtime command class available. + */ + public static final class Builder { + + /** + * Serialized patch command. + */ + private final String patchCommand; + + /** + * Traversal direction for the command. + */ + private final WordTraversalDirection traversalDirection; + + private Builder(final String patchCommand, final WordTraversalDirection traversalDirection) { + this.patchCommand = patchCommand; + this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection"); + } + + /** + * Builds the concrete command instance. + * + * @return compiled command instance + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP + * argument + */ + public CompiledPatchCommand build() { + if (this.patchCommand == null) { + return preserve(this.traversalDirection); + } + + final int patchLength = this.patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { + return preserve(this.traversalDirection); + } + if (patchLength == SINGLE_COMMAND_LENGTH) { + return compileSingle(this.patchCommand.charAt(0), this.patchCommand.charAt(1), + this.traversalDirection); + } + + final int operationCount = patchLength >> 1; + final char[] opcodes = new char[operationCount]; + final int[] operands = new int[operationCount]; + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += SINGLE_COMMAND_LENGTH) { + final int operationIndex = patchIndex >> 1; + final char opcode = this.patchCommand.charAt(patchIndex); + final int operand = compileOperand(opcode, this.patchCommand.charAt(patchIndex + 1)); + if (operand < 0) { + return preserve(this.traversalDirection); + } + opcodes[operationIndex] = opcode; + operands[operationIndex] = operand; + } + + final int lengthDelta = computeLengthDelta(opcodes, operands); + final int minimumSourceLength = this.traversalDirection == WordTraversalDirection.BACKWARD + ? computeBackwardMinimumSourceLength(opcodes, operands) + : computeForwardMinimumSourceLength(opcodes, operands); + return this.traversalDirection == WordTraversalDirection.BACKWARD + ? new BackwardCompoundCommand(this.traversalDirection, opcodes, operands, lengthDelta, + minimumSourceLength) + : new ForwardCompoundCommand(this.traversalDirection, opcodes, operands, lengthDelta, + minimumSourceLength); + } + } + + private static CompiledPatchCommand preserve(final WordTraversalDirection traversalDirection) { + return new PreserveCommand(traversalDirection); + } + + private static CompiledPatchCommand compileSingle(final char opcode, final char argument, + final WordTraversalDirection traversalDirection) { + switch (opcode) { + case DELETE_OPCODE: + final int deleteCount = decodeEncodedCount(argument); + if (deleteCount < MINIMUM_COUNT) { + return preserve(traversalDirection); + } + return traversalDirection == WordTraversalDirection.BACKWARD + ? new DeleteSuffixCommand(traversalDirection, deleteCount) + : new DeletePrefixCommand(traversalDirection, deleteCount); + case INSERT_OPCODE: + return traversalDirection == WordTraversalDirection.BACKWARD + ? new AppendCharacterCommand(traversalDirection, argument) + : new PrependCharacterCommand(traversalDirection, argument); + case REPLACE_OPCODE: + return traversalDirection == WordTraversalDirection.BACKWARD + ? new ReplaceLastCharacterCommand(traversalDirection, argument) + : new ReplaceFirstCharacterCommand(traversalDirection, argument); + case SKIP_OPCODE: + return preserve(traversalDirection); + case NOOP_OPCODE: + if (argument != NOOP_ARGUMENT) { + throw new IllegalArgumentException(MSG_NOOP + argument); + } + return preserve(traversalDirection); + default: + throw new IllegalArgumentException(MSG_OPCODE + opcode); + } + } + + private static int compileOperand(final char opcode, final char argument) { + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = decodeEncodedCount(argument); + return skipCount < MINIMUM_COUNT ? -1 : skipCount; + case DELETE_OPCODE: + final int deleteCount = decodeEncodedCount(argument); + return deleteCount < MINIMUM_COUNT ? -1 : deleteCount; + case INSERT_OPCODE: + case REPLACE_OPCODE: + return argument; + case NOOP_OPCODE: + if (argument != NOOP_ARGUMENT) { + throw new IllegalArgumentException(MSG_NOOP + argument); + } + return -1; + default: + throw new IllegalArgumentException(MSG_OPCODE + opcode); + } + } + + private static int computeLengthDelta(final char[] opcodes, final int[] operands) { + int lengthDelta = 0; + for (int index = 0; index < opcodes.length; index++) { + switch (opcodes[index]) { + case DELETE_OPCODE: + lengthDelta -= operands[index]; + break; + case INSERT_OPCODE: + lengthDelta++; + break; + case SKIP_OPCODE: + case REPLACE_OPCODE: + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return lengthDelta; + } + + private static int computeForwardMinimumSourceLength(final char[] opcodes, final int[] operands) { + int minimumSourceLength = 0; + int position = 0; + int lengthDelta = 0; + for (int index = 0; index < opcodes.length; index++) { + final int operand = operands[index]; + switch (opcodes[index]) { + case SKIP_OPCODE: + position += operand; + break; + case DELETE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position + operand - lengthDelta); + lengthDelta -= operand; + break; + case INSERT_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position - lengthDelta); + lengthDelta++; + position++; + break; + case REPLACE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position + 1 - lengthDelta); + position++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return minimumSourceLength; + } + + private static int computeBackwardMinimumSourceLength(final char[] opcodes, final int[] operands) { + int minimumSourceLength = 0; + int consumedFromEnd = 0; + for (int index = 0; index < opcodes.length; index++) { + final int operand = operands[index]; + switch (opcodes[index]) { + case SKIP_OPCODE: + consumedFromEnd += operand; + break; + case DELETE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd + operand); + consumedFromEnd += operand; + break; + case INSERT_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd); + break; + case REPLACE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd + 1); + consumedFromEnd++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return minimumSourceLength; + } + + private static int decodeEncodedCount(final char argument) { + if (argument < FIRST_COUNT_ARGUMENT) { + return -1; + } + return argument - FIRST_COUNT_ARGUMENT + MINIMUM_COUNT; + } + + private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset) { + if (sourceLength <= 0) { + return; + } + if (source instanceof String sourceString) { + sourceString.getChars(sourceOffset, sourceOffset + sourceLength, output, outputOffset); + return; + } + for (int index = 0; index < sourceLength; index++) { + output[outputOffset + index] = source.charAt(sourceOffset + index); + } + } + + private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int outputLength) { + if (!sameArray(source, output) || sourceLength == 0 || outputLength == 0) { + return; + } + final int sourceEnd = sourceOffset + sourceLength; + final int outputEnd = outputOffset + outputLength; + if (sourceOffset < outputEnd && outputOffset < sourceEnd) { + throw new IllegalArgumentException("source and output ranges must not overlap."); + } + } + + @SuppressWarnings("PMD.CompareObjectsWithEquals") + private static boolean sameArray(final char[] left, final char[] right) { + return left == right; + } + + /** + * Command that preserves the source unchanged. + */ + private static final class PreserveCommand extends CompiledPatchCommand { + + private PreserveCommand(final WordTraversalDirection traversalDirection) { + super(traversalDirection, 0, 0); + } + + @Override + protected String applyNonNull(final String source) { + return source; + } + + @Override + public boolean preservesAllSources() { + return true; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + + /** + * Command that deletes characters from the logical suffix of a backward patch. + */ + private static final class DeleteSuffixCommand extends CompiledPatchCommand { + + /** + * Number of suffix characters deleted. + */ + private final int count; + + private DeleteSuffixCommand(final WordTraversalDirection traversalDirection, final int count) { + super(traversalDirection, -count, 0); + this.count = count; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + return source.substring(0, sourceLength - this.count); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, producedLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, producedLength); + } + } + + /** + * Command that deletes characters from the logical prefix of a forward patch. + */ + private static final class DeletePrefixCommand extends CompiledPatchCommand { + + /** + * Number of prefix characters deleted. + */ + private final int count; + + private DeletePrefixCommand(final WordTraversalDirection traversalDirection, final int count) { + super(traversalDirection, -count, 0); + this.count = count; + } + + @Override + protected String applyNonNull(final String source) { + if (!hasApplicableLength(source.length())) { + return source; + } + return source.substring(this.count); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + final int effectiveOffset = producedLength == sourceLength ? sourceOffset : sourceOffset + this.count; + copySource(source, effectiveOffset, producedLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + final int effectiveOffset = producedLength == sourceLength ? sourceOffset : sourceOffset + this.count; + System.arraycopy(source, effectiveOffset, output, outputOffset, producedLength); + } + } + + /** + * Command that appends one character to a backward patch result. + */ + private static final class AppendCharacterCommand extends CompiledPatchCommand { + + /** + * Appended character. + */ + private final char character; + + private AppendCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 1, 0); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + final char[] target = new char[sourceLength + 1]; + source.getChars(0, sourceLength, target, 0); + target[sourceLength] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + output[outputOffset + sourceLength] = this.character; + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + output[outputOffset + sourceLength] = this.character; + } + } + + /** + * Command that prepends one character to a forward patch result. + */ + private static final class PrependCharacterCommand extends CompiledPatchCommand { + + /** + * Prepended character. + */ + private final char character; + + private PrependCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 1, 0); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + final char[] target = new char[sourceLength + 1]; + target[0] = this.character; + source.getChars(0, sourceLength, target, 1); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + output[outputOffset] = this.character; + copySource(source, sourceOffset, sourceLength, output, outputOffset + 1); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + output[outputOffset] = this.character; + System.arraycopy(source, sourceOffset, output, outputOffset + 1, sourceLength); + } + } + + /** + * Command that replaces the final character of a backward patch result. + */ + private static final class ReplaceLastCharacterCommand extends CompiledPatchCommand { + + /** + * Replacement character. + */ + private final char character; + + private ReplaceLastCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 0, 1); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (sourceLength == 0) { + return source; + } + final char[] target = source.toCharArray(); + target[sourceLength - 1] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + if (sourceLength > 0) { + output[outputOffset + sourceLength - 1] = this.character; + } + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + if (sourceLength > 0) { + output[outputOffset + sourceLength - 1] = this.character; + } + } + } + + /** + * Command that replaces the first character of a forward patch result. + */ + private static final class ReplaceFirstCharacterCommand extends CompiledPatchCommand { + + /** + * Replacement character. + */ + private final char character; + + private ReplaceFirstCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 0, 1); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + if (source.isEmpty()) { + return source; + } + final char[] target = source.toCharArray(); + target[0] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + if (sourceLength > 0) { + output[outputOffset] = this.character; + } + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + if (sourceLength > 0) { + output[outputOffset] = this.character; + } + } + } + + /** + * Compound command that applies atomic operations in backward traversal order. + */ + private static final class BackwardCompoundCommand extends CompiledPatchCommand { + + /** + * Operation opcodes in serialized order. + */ + private final char[] opcodes; + + /** + * Operation counts or character operands in serialized order. + */ + private final int[] operands; + + private BackwardCompoundCommand(final WordTraversalDirection traversalDirection, final char[] opcodes, + final int[] operands, final int lengthDelta, final int minimumSourceLength) { + super(traversalDirection, lengthDelta, minimumSourceLength); + this.opcodes = opcodes; + this.operands = operands; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + final int producedLength = computeAppliedLength(sourceLength); + final char[] target = new char[producedLength]; + writeSequence(source, 0, sourceLength, target, 0, producedLength); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + writeSequence(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + writeArray(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + private void writeSequence(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeBackwardSequence(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + } + + private void writeArray(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeBackwardArray(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + } + + /** + * Compound command that applies atomic operations in forward traversal order. + */ + private static final class ForwardCompoundCommand extends CompiledPatchCommand { + + /** + * Operation opcodes in serialized order. + */ + private final char[] opcodes; + + /** + * Operation counts or character operands in serialized order. + */ + private final int[] operands; + + private ForwardCompoundCommand(final WordTraversalDirection traversalDirection, final char[] opcodes, + final int[] operands, final int lengthDelta, final int minimumSourceLength) { + super(traversalDirection, lengthDelta, minimumSourceLength); + this.opcodes = opcodes; + this.operands = operands; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + final int producedLength = computeAppliedLength(sourceLength); + final char[] target = new char[producedLength]; + writeSequence(source, 0, sourceLength, target, 0, producedLength); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + writeSequence(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + writeArray(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + private void writeSequence(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeForwardSequence(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + } + + private void writeArray(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeForwardArray(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + } + + private static boolean writeForwardSequence(final char[] opcodes, final int[] operands, + final CharSequence source, final int sourceOffset, final int sourceLength, final char[] output, + final int outputOffset, final int producedLength) { + int currentLength = sourceLength; + int position = 0; + int sourceIndex = 0; + int outputIndex = 0; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceLength - sourceIndex); + copySource(source, sourceOffset + sourceIndex, skipCount, output, outputOffset + outputIndex); + sourceIndex += skipCount; + outputIndex += skipCount; + position = position + operand - 1; + break; + case DELETE_OPCODE: + if (position < 0 || position > currentLength) { + return false; + } + final int deletedLength = Math.min(operand, currentLength - position); + if (sourceIndex + deletedLength > sourceLength) { + return false; + } + sourceIndex += deletedLength; + currentLength -= deletedLength; + position--; + break; + case INSERT_OPCODE: + if (position < 0 || position > currentLength || outputIndex >= producedLength) { + return false; + } + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + currentLength++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceIndex >= sourceLength + || outputIndex >= producedLength) { + return false; + } + sourceIndex++; + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position++; + } + final int remainingLength = sourceLength - sourceIndex; + if (remainingLength > producedLength - outputIndex) { + return false; + } + copySource(source, sourceOffset + sourceIndex, remainingLength, output, outputOffset + outputIndex); + return outputIndex + remainingLength == producedLength; + } + + private static boolean writeForwardArray(final char[] opcodes, final int[] operands, final char[] source, + final int sourceOffset, final int sourceLength, final char[] output, final int outputOffset, + final int producedLength) { + int currentLength = sourceLength; + int position = 0; + int sourceIndex = 0; + int outputIndex = 0; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceLength - sourceIndex); + System.arraycopy(source, sourceOffset + sourceIndex, output, outputOffset + outputIndex, + skipCount); + sourceIndex += skipCount; + outputIndex += skipCount; + position = position + operand - 1; + break; + case DELETE_OPCODE: + if (position < 0 || position > currentLength) { + return false; + } + final int deletedLength = Math.min(operand, currentLength - position); + if (sourceIndex + deletedLength > sourceLength) { + return false; + } + sourceIndex += deletedLength; + currentLength -= deletedLength; + position--; + break; + case INSERT_OPCODE: + if (position < 0 || position > currentLength || outputIndex >= producedLength) { + return false; + } + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + currentLength++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceIndex >= sourceLength + || outputIndex >= producedLength) { + return false; + } + sourceIndex++; + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position++; + } + final int remainingLength = sourceLength - sourceIndex; + if (remainingLength > producedLength - outputIndex) { + return false; + } + System.arraycopy(source, sourceOffset + sourceIndex, output, outputOffset + outputIndex, remainingLength); + return outputIndex + remainingLength == producedLength; + } + + private static boolean writeBackwardSequence(final char[] opcodes, final int[] operands, + final CharSequence source, final int sourceOffset, final int sourceLength, final char[] output, + final int outputOffset, final int producedLength) { + int currentLength = sourceLength; + int position = sourceLength - 1; + int sourceEnd = sourceLength; + int outputEnd = producedLength; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceEnd); + sourceEnd -= skipCount; + outputEnd -= skipCount; + if (outputEnd < 0) { + return false; + } + copySource(source, sourceOffset + sourceEnd, skipCount, output, outputOffset + outputEnd); + position = position - operand + 1; + break; + case DELETE_OPCODE: + final int deleteEndExclusive = position + 1; + position -= operand - 1; + if (position < 0 || position > currentLength || position > deleteEndExclusive) { + return false; + } + final int deletedLength = Math.min(deleteEndExclusive, currentLength) - position; + if (sourceEnd < deletedLength) { + return false; + } + sourceEnd -= deletedLength; + currentLength -= deletedLength; + break; + case INSERT_OPCODE: + if (position < -1 || position >= currentLength || outputEnd <= 0) { + return false; + } + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + currentLength++; + position++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceEnd <= 0 || outputEnd <= 0) { + return false; + } + sourceEnd--; + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position--; + } + if (sourceEnd != outputEnd) { + return false; + } + copySource(source, sourceOffset, sourceEnd, output, outputOffset); + return true; + } + + private static boolean writeBackwardArray(final char[] opcodes, final int[] operands, final char[] source, + final int sourceOffset, final int sourceLength, final char[] output, final int outputOffset, + final int producedLength) { + int currentLength = sourceLength; + int position = sourceLength - 1; + int sourceEnd = sourceLength; + int outputEnd = producedLength; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceEnd); + sourceEnd -= skipCount; + outputEnd -= skipCount; + if (outputEnd < 0) { + return false; + } + System.arraycopy(source, sourceOffset + sourceEnd, output, outputOffset + outputEnd, skipCount); + position = position - operand + 1; + break; + case DELETE_OPCODE: + final int deleteEndExclusive = position + 1; + position -= operand - 1; + if (position < 0 || position > currentLength || position > deleteEndExclusive) { + return false; + } + final int deletedLength = Math.min(deleteEndExclusive, currentLength) - position; + if (sourceEnd < deletedLength) { + return false; + } + sourceEnd -= deletedLength; + currentLength -= deletedLength; + break; + case INSERT_OPCODE: + if (position < -1 || position >= currentLength || outputEnd <= 0) { + return false; + } + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + currentLength++; + position++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceEnd <= 0 || outputEnd <= 0) { + return false; + } + sourceEnd--; + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position--; + } + if (sourceEnd != outputEnd) { + return false; + } + System.arraycopy(source, sourceOffset, output, outputOffset, sourceEnd); + return true; + } +} diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrie.java b/src/main/java/org/egothor/stemmer/FrequencyTrie.java index 9c1cae4..9ff0e11 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrie.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrie.java @@ -89,7 +89,7 @@ import org.egothor.stemmer.trie.ReductionSignature; * * @param value type */ -@SuppressWarnings("PMD.CyclomaticComplexity") +@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.CouplingBetweenObjects" }) public final class FrequencyTrie { /** @@ -105,7 +105,7 @@ public final class FrequencyTrie { /** * Version of the canonical fingerprint input format. */ - private static final int FINGERPRINT_FORMAT_VERSION = 1; + private static final int FINGERPRINT_FORMAT_VERSION = 2; /** * Root node of the compiled read-only trie. @@ -169,7 +169,7 @@ public final class FrequencyTrie { /** * Binary format version. */ - private static final int STREAM_VERSION = 5; + private static final int STREAM_VERSION = 6; /** * Version where traversal-direction ordinal is persisted. @@ -186,6 +186,16 @@ public final class FrequencyTrie { */ private static final int CASE_VERSION = 4; + /** + * Version where the persisted metadata switched to a text block. + */ + private static final int TEXT_METADATA_VERSION = 5; + + /** + * Version where contracted accepting nodes are persisted. + */ + private static final int ACCEPTING_NODE_VERSION = 6; + /** * Argument name for lookup keys. */ @@ -260,6 +270,21 @@ public final class FrequencyTrie { this.emptyValues = arrayFactory.apply(0); } + /** + * Creates a trie from an already compiled root. + * + * @param arrayFactory array factory + * @param root compiled root + * @param metadata trie metadata + * @param value type + * @return trie instance + */ + /* default */ static FrequencyTrie fromCompiled(final IntFunction arrayFactory, + final CompiledNode root, + final TrieMetadata metadata) { + return new FrequencyTrie<>(arrayFactory, root, metadata); + } + /** * Returns the most frequent value stored at the node addressed by the supplied * key. @@ -292,6 +317,63 @@ public final class FrequencyTrie { return orderedValues[0]; } + /** + * Returns the preferred value for an already-normalized key. + * + *

    + * This method bypasses {@link TrieMetadata#caseProcessingMode()} and + * {@link TrieMetadata#diacriticProcessingMode()}. Callers must supply input + * normalized exactly as required by this trie's metadata. It is intended for + * hot paths where normalization is guaranteed by an upstream tokenizer or + * benchmark corpus and repeated lookup-time normalization would be redundant. + *

    + * + * @param key already-normalized key to resolve + * @return most frequent value, or {@code null} if the key does not exist or no + * value is stored at the addressed node + * @throws NullPointerException if {@code key} is {@code null} + */ + public V getNormalized(final CharSequence key) { + Objects.requireNonNull(key, ARG_KEY); + final CompiledNode node = findNode(key); + if (node == null) { + return null; + } + final V[] orderedValues = node.orderedValues(); + if (orderedValues.length == 0) { + return null; + } + return orderedValues[0]; + } + + /** + * Returns the preferred value for an already-normalized {@link String} key. + * + *

    + * This overload keeps high-volume string lookup on a monomorphic path and + * avoids the {@link CharSequence} dispatch used by the general overload. + * Callers must supply input normalized exactly as required by this trie's + * metadata. + *

    + * + * @param key already-normalized key to resolve + * @return most frequent value, or {@code null} if the key does not exist or no + * value is stored at the addressed node + * @throws NullPointerException if {@code key} is {@code null} + */ + public V getNormalizedString(final String key) { + Objects.requireNonNull(key, ARG_KEY); + final CompiledNode node = findNode(key); + if (node == null) { + return null; + } + final V[] orderedValues = node.orderedValues(); + if (orderedValues.length == 0) { + return null; + } + return orderedValues[0]; + } + /** * Returns all values stored at the node addressed by the supplied key, ordered * by descending frequency. @@ -745,6 +827,7 @@ public final class FrequencyTrie { */ private static void writeNode(final DataOutputStream dataOutput, final ValueStreamCodec valueCodec, final CompiledNode node, final Map, Integer> nodeIds) throws IOException { + dataOutput.writeBoolean(node.acceptsRemainingInput()); dataOutput.writeInt(node.edgeLabels().length); for (int index = 0; index < node.edgeLabels().length; index++) { dataOutput.writeChar(node.edgeLabels()[index]); @@ -777,6 +860,7 @@ public final class FrequencyTrie { final V[] values = node.orderedValues(); final int[] counts = node.orderedCounts(); + updateInt(messageDigest, node.acceptsRemainingInput() ? 1 : 0); updateInt(messageDigest, edgeLabels.length); for (char edgeLabel : edgeLabels) { updateInt(messageDigest, edgeLabel); @@ -863,7 +947,7 @@ public final class FrequencyTrie { final TrieMetadata sourceMetadata = readMetadata(dataInput, version); final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX; final CompiledNode[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount, - effectiveMaxExpandedIndex); + effectiveMaxExpandedIndex, version); final CompiledNode rootNode = nodes[rootNodeId]; if (LOGGER.isLoggable(Level.FINE)) { @@ -880,8 +964,8 @@ public final class FrequencyTrie { private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version) throws IOException { - if (version == STREAM_VERSION) { - return readTextMetadata(dataInput); + if (version >= TEXT_METADATA_VERSION) { + return readTextMetadata(dataInput, version); } final WordTraversalDirection traversalDirection = readTraversalDirection(dataInput, version); @@ -898,9 +982,10 @@ public final class FrequencyTrie { caseProcessingMode); } - private static TrieMetadata readTextMetadata(final DataInputStream dataInput) throws IOException { + private static TrieMetadata readTextMetadata(final DataInputStream dataInput, final int version) + throws IOException { try { - return TrieMetadata.fromTextBlock(STREAM_VERSION, dataInput.readUTF()); + return TrieMetadata.fromTextBlock(version, dataInput.readUTF()); } catch (IllegalArgumentException exception) { throw new IOException("Invalid metadata block.", exception); } @@ -936,14 +1021,19 @@ public final class FrequencyTrie { private static CompiledNode[] readNodes(final DataInputStream dataInput, final IntFunction arrayFactory, final ValueStreamCodec valueCodec, final int nodeCount, - final int maxExpandedIndex) throws IOException { + final int maxExpandedIndex, final int version) throws IOException { final char[][] edgeLabelsByNode = new char[nodeCount][]; final int[][] childNodeIdsByNode = new int[nodeCount][]; @SuppressWarnings("unchecked") final V[][] orderedValuesByNode = (V[][]) new Object[nodeCount][]; final int[][] orderedCountsByNode = new int[nodeCount][]; + final boolean[] acceptsRemainingInputByNode = new boolean[nodeCount]; for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { + if (version >= ACCEPTING_NODE_VERSION) { + acceptsRemainingInputByNode[nodeIndex] = dataInput.readBoolean(); + } + final int edgeCount = dataInput.readInt(); if (edgeCount < 0) { throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount); @@ -963,6 +1053,12 @@ public final class FrequencyTrie { if (valueCount < 0) { throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount); } + if (acceptsRemainingInputByNode[nodeIndex] && edgeCount != 0) { + throw new IOException("Accepting node " + nodeIndex + " cannot have child edges."); + } + if (acceptsRemainingInputByNode[nodeIndex] && valueCount == 0) { + throw new IOException("Accepting node " + nodeIndex + " must store at least one value."); + } orderedValuesByNode[nodeIndex] = arrayFactory.apply(valueCount); orderedCountsByNode[nodeIndex] = new int[valueCount]; @@ -983,7 +1079,7 @@ public final class FrequencyTrie { for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { nodes[nodeIndex] = resolveNode(nodeIndex, edgeLabelsByNode, childNodeIdsByNode, orderedValuesByNode, - orderedCountsByNode, nodes, inProgress, maxExpandedIndex); + orderedCountsByNode, acceptsRemainingInputByNode, nodes, inProgress, maxExpandedIndex); } return nodes; @@ -991,8 +1087,8 @@ public final class FrequencyTrie { private static CompiledNode resolveNode(final int nodeIndex, final char[][] edgeLabelsByNode, final int[][] childNodeIdsByNode, final V[][] orderedValuesByNode, final int[][] orderedCountsByNode, - final CompiledNode[] nodes, final boolean[] inProgress, final int maxExpandedIndex) - throws IOException { + final boolean[] acceptsRemainingInputByNode, final CompiledNode[] nodes, + final boolean[] inProgress, final int maxExpandedIndex) throws IOException { final CompiledNode cachedNode = nodes[nodeIndex]; if (cachedNode != null) { return cachedNode; @@ -1017,11 +1113,12 @@ public final class FrequencyTrie { + ": " + childNodeId); } children[edgeIndex] = resolveNode(childNodeId, edgeLabelsByNode, childNodeIdsByNode, - orderedValuesByNode, orderedCountsByNode, nodes, inProgress, maxExpandedIndex); + orderedValuesByNode, orderedCountsByNode, acceptsRemainingInputByNode, nodes, inProgress, + maxExpandedIndex); } final CompiledNode node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex], - maxExpandedIndex, orderedCountsByNode[nodeIndex]); + acceptsRemainingInputByNode[nodeIndex], maxExpandedIndex, orderedCountsByNode[nodeIndex]); nodes[nodeIndex] = node; return node; } finally { @@ -1047,7 +1144,30 @@ public final class FrequencyTrie { * @return compiled node, or {@code null} if the path does not exist */ private CompiledNode findNode(final String key) { - return findNode((CharSequence) key); + CompiledNode current = this.root; + if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) { + for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) { + if (current.acceptsRemainingInput()) { + return current; + } + current = current.findChild(key.charAt(traversalOffset)); + if (current == null) { + return null; + } + } + return current; + } + + for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) { + if (current.acceptsRemainingInput()) { + return current; + } + current = current.findChild(key.charAt(traversalOffset)); + if (current == null) { + return null; + } + } + return current; } /** @@ -1060,6 +1180,9 @@ public final class FrequencyTrie { CompiledNode current = this.root; if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) { for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) { + if (current.acceptsRemainingInput()) { + return current; + } current = current.findChild(key.charAt(traversalOffset)); if (current == null) { return null; @@ -1069,6 +1192,9 @@ public final class FrequencyTrie { } for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) { + if (current.acceptsRemainingInput()) { + return current; + } current = current.findChild(key.charAt(traversalOffset)); if (current == null) { return null; @@ -1089,6 +1215,9 @@ public final class FrequencyTrie { CompiledNode current = this.root; if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) { for (int traversalOffset = offset + length - 1; traversalOffset >= offset; traversalOffset--) { + if (current.acceptsRemainingInput()) { + return current; + } current = current.findChild(key[traversalOffset]); if (current == null) { return null; @@ -1099,6 +1228,9 @@ public final class FrequencyTrie { final int endExclusive = offset + length; for (int traversalOffset = offset; traversalOffset < endExclusive; traversalOffset++) { + if (current.acceptsRemainingInput()) { + return current; + } current = current.findChild(key[traversalOffset]); if (current == null) { return null; @@ -1535,21 +1667,31 @@ public final class FrequencyTrie { * @return canonical reduced node */ private ReducedNode reduce(final MutableNode source, final ReductionContext context) { - final Map> reducedChildren = new LinkedHashMap<>(); + Map> reducedChildren = new LinkedHashMap<>(); for (Map.Entry> childEntry : source.children().entrySet()) { final ReducedNode reducedChild = reduce(childEntry.getValue(), context); reducedChildren.put(childEntry.getKey(), reducedChild); } - final Map localCounts = copyCounts(source.valueCounts()); + Map localCounts = copyCounts(source.valueCounts()); + boolean acceptsRemainingInput = false; + if (context.settings().contractUniformSubtrees()) { + final Map contractedCounts = contractUniformSubtree(localCounts, reducedChildren); + if (!contractedCounts.isEmpty()) { + localCounts = contractedCounts; + reducedChildren = Collections.emptyMap(); + acceptsRemainingInput = true; + } + } + final LocalValueSummary localSummary = LocalValueSummary.of(localCounts, this.arrayFactory); final ReductionSignature signature = ReductionSignature.create(localSummary, reducedChildren, - context.settings()); + context.settings(), acceptsRemainingInput); ReducedNode canonical = context.lookup(signature); if (canonical == null) { - canonical = new ReducedNode<>(signature, localCounts, reducedChildren); + canonical = new ReducedNode<>(signature, localCounts, reducedChildren, acceptsRemainingInput); context.register(signature, canonical); return canonical; } @@ -1560,6 +1702,64 @@ public final class FrequencyTrie { return canonical; } + /** + * Returns aggregated local counts when the supplied internal subtree contains + * one uniform value, otherwise {@code null}. + * + * @param localCounts local counts at the current node + * @param reducedChildren already reduced children + * @return single-value aggregate for a uniform non-leaf subtree, otherwise an + * empty map + */ + private Map contractUniformSubtree(final Map localCounts, + final Map> reducedChildren) { + if (reducedChildren.isEmpty()) { + return Collections.emptyMap(); + } + + V uniformValue = null; + boolean valueSeen = false; + + if (!localCounts.isEmpty()) { + if (localCounts.size() != SINGLE_VALUE_COUNT) { + return Collections.emptyMap(); + } + final Map.Entry localEntry = localCounts.entrySet().iterator().next(); + uniformValue = localEntry.getKey(); + valueSeen = true; + } + + for (ReducedNode child : reducedChildren.values()) { + if (!isSingleValueLeaf(child)) { + return Collections.emptyMap(); + } + final Map.Entry childEntry = child.localCounts().entrySet().iterator().next(); + if (valueSeen && !Objects.equals(uniformValue, childEntry.getKey())) { + return Collections.emptyMap(); + } + uniformValue = childEntry.getKey(); + valueSeen = true; + } + + if (!valueSeen) { + return Collections.emptyMap(); + } + + final Map contractedCounts = new LinkedHashMap<>(SINGLE_VALUE_COUNT); + contractedCounts.put(uniformValue, SINGLE_VALUE_COUNT); + return contractedCounts; + } + + /** + * Returns whether the reduced node is a leaf with exactly one stored value. + * + * @param node node to inspect + * @return {@code true} when the node can participate in uniform contraction + */ + private boolean isSingleValueLeaf(final ReducedNode node) { + return node.children().isEmpty() && node.localCounts().size() == SINGLE_VALUE_COUNT; + } + /** * Freezes a reduced node into an immutable compiled node. * @@ -1592,7 +1792,7 @@ public final class FrequencyTrie { } final CompiledNode frozen = new CompiledNode<>(edges, childNodes, localSummary.orderedValues(), - this.maxExpandedIndex, localSummary.orderedCounts()); + reducedNode.acceptsRemainingInput(), this.maxExpandedIndex, localSummary.orderedCounts()); cache.put(reducedNode, frozen); return frozen; } diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java b/src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java index ef4b98e..372d743 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java @@ -30,7 +30,10 @@ ******************************************************************************/ package org.egothor.stemmer; +import java.util.IdentityHashMap; +import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.function.IntFunction; import java.util.logging.Level; import java.util.logging.Logger; @@ -115,6 +118,63 @@ public final class FrequencyTrieBuilders { return copyOf(source, arrayFactory, ReductionSettings.withDefaults(reductionMode)); } + /** + * Reconstructs a compiled trie with every stored value transformed to another + * value type. + * + *

    + * The method preserves logical keys, local value counts, trie metadata, and the + * supplied reduction settings. It is intended for runtime specialization, such + * as replacing serialized patch-command strings with precompiled patch command + * objects without changing the persisted binary trie format. + *

    + * + * @param source source compiled trie + * @param arrayFactory array factory for mapped values + * @param reductionSettings reduction settings for the mapped trie + * @param valueMapper value mapping function + * @param source value type + * @param target value type + * @return compiled trie containing mapped values + * @throws NullPointerException if any argument is {@code null} + */ + public static FrequencyTrie mapValues(final FrequencyTrie source, + final IntFunction arrayFactory, final ReductionSettings reductionSettings, + final Function valueMapper) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(arrayFactory, "arrayFactory"); + Objects.requireNonNull(reductionSettings, "reductionSettings"); + Objects.requireNonNull(valueMapper, "valueMapper"); + + final Map, CompiledNode> cache = new IdentityHashMap<>(); + final CompiledNode mappedRoot = mapCompiledNode(source.root(), arrayFactory, valueMapper, cache); + final TrieMetadata metadata = TrieMetadata.forCompilation(source.traversalDirection(), reductionSettings, + source.metadata().diacriticProcessingMode(), source.metadata().caseProcessingMode()); + + LOGGER.log(Level.FINE, "Mapped compiled trie values to a specialized value type."); + return FrequencyTrie.fromCompiled(arrayFactory, mappedRoot, metadata); + } + + /** + * Reconstructs a compiled trie with every stored value transformed to another + * value type using default settings for the supplied reduction mode. + * + * @param source source compiled trie + * @param arrayFactory array factory for mapped values + * @param reductionMode reduction mode for the mapped trie + * @param valueMapper value mapping function + * @param source value type + * @param target value type + * @return compiled trie containing mapped values + * @throws NullPointerException if any argument is {@code null} + */ + public static FrequencyTrie mapValues(final FrequencyTrie source, + final IntFunction arrayFactory, final ReductionMode reductionMode, + final Function valueMapper) { + Objects.requireNonNull(reductionMode, "reductionMode"); + return mapValues(source, arrayFactory, ReductionSettings.withDefaults(reductionMode), valueMapper); + } + /** * Copies one compiled node and all reachable descendants into the target * builder. @@ -138,4 +198,43 @@ public final class FrequencyTrieBuilders { keyBuilder.setLength(keyBuilder.length() - 1); } } + + /** + * Maps one compiled node graph while preserving canonical sharing and accepting + * leaf semantics. + * + * @param node source node + * @param arrayFactory target value array factory + * @param valueMapper value mapper + * @param cache identity cache for shared compiled nodes + * @param source value type + * @param target value type + * @return mapped compiled node + */ + private static CompiledNode mapCompiledNode(final CompiledNode node, + final IntFunction arrayFactory, final Function valueMapper, + final Map, CompiledNode> cache) { + final CompiledNode existing = cache.get(node); + if (existing != null) { + return existing; + } + + final CompiledNode[] sourceChildren = node.children(); + @SuppressWarnings("unchecked") + final CompiledNode[] mappedChildren = new CompiledNode[sourceChildren.length]; + for (int childIndex = 0; childIndex < sourceChildren.length; childIndex++) { + mappedChildren[childIndex] = mapCompiledNode(sourceChildren[childIndex], arrayFactory, valueMapper, cache); + } + + final S[] sourceValues = node.orderedValues(); + final T[] mappedValues = arrayFactory.apply(sourceValues.length); + for (int valueIndex = 0; valueIndex < sourceValues.length; valueIndex++) { + mappedValues[valueIndex] = valueMapper.apply(sourceValues[valueIndex]); + } + + final CompiledNode mapped = new CompiledNode<>(node.edgeLabels().clone(), mappedChildren, mappedValues, + node.acceptsRemainingInput(), CompiledNode.DEFAULT_MAX_EXPANDED_INDEX, node.orderedCounts().clone()); + cache.put(node, mapped); + return mapped; + } } diff --git a/src/main/java/org/egothor/stemmer/PatchCommandEncoder.java b/src/main/java/org/egothor/stemmer/PatchCommandEncoder.java index 4004efb..db8a01f 100644 --- a/src/main/java/org/egothor/stemmer/PatchCommandEncoder.java +++ b/src/main/java/org/egothor/stemmer/PatchCommandEncoder.java @@ -70,16 +70,6 @@ import java.util.concurrent.locks.ReentrantLock; @SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" }) public final class PatchCommandEncoder { - /** - * Backward direction apply strategy with no runtime direction branching. - */ - private static final ApplyStrategy BACKWARD_APPLY_STRATEGY = PatchCommandEncoder::applyBackward; - - /** - * Forward direction apply strategy with no runtime direction branching. - */ - private static final ApplyStrategy FORWARD_APPLY_STRATEGY = PatchCommandEncoder::applyForward; - /** * Serialized opcode for deleting one or more characters. */ @@ -175,9 +165,9 @@ public final class PatchCommandEncoder { private final WordTraversalDirection traversalDirection; /** - * Direction-specialized patch apply strategy. + * Whether this instance applies patch commands in backward traversal order. */ - private final ApplyStrategy applyStrategy; + private final boolean backwardTraversal; /** * Currently allocated source dimension of reusable matrices. @@ -222,21 +212,6 @@ public final class PatchCommandEncoder { MATCH } - /** - * Direction-specialized patch application strategy. - */ - @FunctionalInterface - private interface ApplyStrategy { - /** - * Applies the command. - * - * @param source original text - * @param patchCommand patch command - * @return final text after applying the command - */ - String apply(String source, String patchCommand); - } - private PatchCommandEncoder(final Builder builder) { this.traversalDirection = Objects.requireNonNull(builder.traversalDirection, "traversalDirection"); final int insertCost = builder.insertCost; @@ -260,7 +235,7 @@ public final class PatchCommandEncoder { this.deleteCost = deleteCost; this.replaceCost = replaceCost; this.matchCost = matchCost; - this.applyStrategy = applyStrategyFor(this.traversalDirection); + this.backwardTraversal = this.traversalDirection == WordTraversalDirection.BACKWARD; this.sourceCapacity = 0; this.targetCapacity = 0; this.costMatrix = new int[0][0]; @@ -304,19 +279,43 @@ public final class PatchCommandEncoder { * direction. * *

    - * This is the branch-free instance-level fast path for repeated patch - * application in a known traversal direction. + * This is the instance-level fast path for repeated patch application in a + * known traversal direction. It avoids the static API null and direction + * validation path and calls the selected decoder directly. *

    * * @param source original source word * @param patchCommand compact patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should compile + * {@code patchCommand} once through {@link #compile(String)} and + * reuse {@link CompiledPatchCommand#apply(String)}. The + * String-based application path reparses the patch command on every + * call and is kept only for source compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public String applyWithConfiguredDirection(final String source, final String patchCommand) { if (source == null) { return null; } - return this.applyStrategy.apply(source, patchCommand); + if (this.backwardTraversal) { + return applyBackwardNonNull(source, patchCommand); + } + return applyForwardNonNull(source, patchCommand); + } + + /** + * Compiles a patch command for repeated application with this encoder + * instance traversal direction. + * + * @param patchCommand compact patch command + * @return immutable compiled patch command + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public CompiledPatchCommand compile(final String patchCommand) { + return CompiledPatchCommand.compile(patchCommand, this.traversalDirection); } /** @@ -326,7 +325,14 @@ public final class PatchCommandEncoder { * @param source original source word * @param patchCommand compact patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should use + * {@link CompiledPatchCommand#compile(String, WordTraversalDirection)} + * once and then reuse {@link CompiledPatchCommand#apply(String)}. + * This method repeatedly interprets the serialized patch-command + * string and is retained only for compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static String apply(final String source, final String patchCommand) { return apply(source, patchCommand, WordTraversalDirection.BACKWARD); } @@ -343,14 +349,41 @@ public final class PatchCommandEncoder { * @param patchCommand compact patch command * @param traversalDirection traversal direction used by the patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should use + * {@link CompiledPatchCommand#compile(String, WordTraversalDirection)} + * once and then reuse {@link CompiledPatchCommand#apply(String)}. + * This method repeatedly interprets the serialized patch-command + * string and is retained only for compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static String apply(final String source, final String patchCommand, final WordTraversalDirection traversalDirection) { Objects.requireNonNull(traversalDirection, "traversalDirection"); if (source == null) { return null; } - return applyStrategyFor(traversalDirection).apply(source, patchCommand); + if (traversalDirection == WordTraversalDirection.BACKWARD) { + return applyBackwardNonNull(source, patchCommand); + } + return applyForwardNonNull(source, patchCommand); + } + + /** + * Compiles a patch command for repeated application with the supplied + * traversal direction. + * + * @param patchCommand compact patch command + * @param traversalDirection traversal direction used by the patch command + * @return immutable compiled patch command + * @throws NullPointerException if {@code traversalDirection} is + * {@code null} + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public static CompiledPatchCommand compile(final String patchCommand, + final WordTraversalDirection traversalDirection) { + return CompiledPatchCommand.compile(patchCommand, traversalDirection); } /** @@ -371,7 +404,15 @@ public final class PatchCommandEncoder { * @param outputLength writable output capacity * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} * when {@code outputLength} is too small + * @deprecated Since 2.3.0. Compile {@code patchCommand} once through + * {@link #compile(String, WordTraversalDirection)} and call + * {@link CompiledPatchCommand#applyTo(CharSequence, char[], int, int)} + * or + * {@link CompiledPatchCommand#applyTo(CharSequence, int, int, char[], int, int)}. + * This String-based method reparses patch commands on every call and + * is kept only for compatibility before the 3.0.0 migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static int applyTo(final CharSequence source, final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset, final int outputLength) { @@ -405,7 +446,13 @@ public final class PatchCommandEncoder { * when {@code outputLength} is too small * @throws IllegalArgumentException when source and output ranges overlap in the * same array + * @deprecated Since 2.3.0. Compile {@code patchCommand} once through + * {@link #compile(String, WordTraversalDirection)} and call + * {@link CompiledPatchCommand#applyTo(char[], int, int, char[], int, int)}. + * This String-based method reparses patch commands on every call and + * is kept only for compatibility before the 3.0.0 migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength, final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset, final int outputLength) { @@ -484,36 +531,33 @@ public final class PatchCommandEncoder { /** * Applies a patch command using the historical backward Egothor semantics. * - * @param source original source word in legacy backward logical space + * @param source non-null original source word in legacy backward logical + * space * @param patchCommand compact patch command - * @return transformed word, or {@code null} when {@code source} is {@code null} + * @return transformed word */ - private static String applyBackward(final String source, final String patchCommand) { - if (source == null) { - return null; - } - if (patchCommand == null || patchCommand.isEmpty()) { + private static String applyBackwardNonNull(final String source, final String patchCommand) { + if (patchCommand == null) { return source; } - if (NOOP_PATCH.equals(patchCommand)) { + final int patchLength = patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { return source; } - if ((patchCommand.length() & 1) != 0) { - return source; - } - if (patchCommand.length() == 2) { + if (patchLength == 2) { return applySingleBackwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1)); } - final StringBuilder result = new StringBuilder(source); - if (result.isEmpty()) { - return applyBackwardToEmptySource(result, patchCommand); + if (source.isEmpty()) { + return applyBackwardToEmptySource(patchCommand); } + final StringBuilder result = new StringBuilder(source); + int position = result.length() - 1; try { - for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); final char argument = patchCommand.charAt(patchIndex + 1); @@ -547,12 +591,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return source; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } position--; @@ -567,36 +611,32 @@ public final class PatchCommandEncoder { /** * Applies a patch command using forward traversal semantics. * - * @param source original source word + * @param source non-null original source word * @param patchCommand compact patch command - * @return transformed word, or {@code null} when {@code source} is {@code null} + * @return transformed word */ - private static String applyForward(final String source, final String patchCommand) { - if (source == null) { - return null; - } - if (patchCommand == null || patchCommand.isEmpty()) { + private static String applyForwardNonNull(final String source, final String patchCommand) { + if (patchCommand == null) { return source; } - if (NOOP_PATCH.equals(patchCommand)) { + final int patchLength = patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { return source; } - if ((patchCommand.length() & 1) != 0) { - return source; - } - if (patchCommand.length() == 2) { + if (patchLength == 2) { return applySingleForwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1)); } - final StringBuilder result = new StringBuilder(source); - if (result.isEmpty()) { - return applyForwardToEmptySource(result, patchCommand); + if (source.isEmpty()) { + return applyForwardToEmptySource(patchCommand); } + final StringBuilder result = new StringBuilder(source); + int position = 0; try { - for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); final char argument = patchCommand.charAt(patchIndex + 1); @@ -628,12 +668,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return source; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } position++; @@ -751,12 +791,12 @@ public final class PatchCommandEncoder { * behavior for index-invalid commands. *

    * - * @param result empty result builder * @param patchCommand compact patch command * @return transformed word, or the original empty word when the patch is * malformed */ - private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) { + private static String applyBackwardToEmptySource(final String patchCommand) { + final StringBuilder result = new StringBuilder(patchCommand.length() >> 1); try { for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); @@ -774,12 +814,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return ""; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } } } catch (IndexOutOfBoundsException exception) { @@ -792,12 +832,12 @@ public final class PatchCommandEncoder { /** * Applies a forward patch command to an empty source word. * - * @param result empty result builder * @param patchCommand compact patch command * @return transformed word, or the original empty word when the patch is * malformed */ - private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) { + private static String applyForwardToEmptySource(final String patchCommand) { + final StringBuilder result = new StringBuilder(patchCommand.length() >> 1); try { for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); @@ -815,12 +855,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return ""; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } } } catch (IndexOutOfBoundsException exception) { @@ -1535,16 +1575,6 @@ public final class PatchCommandEncoder { } } - /** - * Returns the direction-specialized apply strategy. - * - * @param traversalDirection requested traversal direction - * @return branch-free apply strategy for that direction - */ - private static ApplyStrategy applyStrategyFor(final WordTraversalDirection traversalDirection) { - return traversalDirection == WordTraversalDirection.BACKWARD ? BACKWARD_APPLY_STRATEGY : FORWARD_APPLY_STRATEGY; - } - /** * Decodes a compact count argument used by skip and delete instructions. * diff --git a/src/main/java/org/egothor/stemmer/ReductionSettings.java b/src/main/java/org/egothor/stemmer/ReductionSettings.java index d6e47dd..8d8b1cb 100644 --- a/src/main/java/org/egothor/stemmer/ReductionSettings.java +++ b/src/main/java/org/egothor/stemmer/ReductionSettings.java @@ -42,10 +42,13 @@ import java.util.Objects; * @param reductionMode reduction mode * @param dominantWinnerMinPercent minimum dominant winner percentage * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio + * @param contractUniformSubtrees whether compilation may contract a subtree + * whose reachable terminal values all contain + * the same single value */ @SuppressWarnings("PMD.LongVariable") public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerMinPercent, - int dominantWinnerOverSecondRatio) { + int dominantWinnerOverSecondRatio, boolean contractUniformSubtrees) { /** * Default minimum dominant winner percentage. @@ -65,12 +68,14 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM * the inclusive range {@code 1..100} * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio, must * be at least {@code 1} + * @param contractUniformSubtrees whether uniform subtrees may be + * contracted into accepting leaves * @throws NullPointerException if {@code reductionMode} is {@code null} * @throws IllegalArgumentException if any numeric value is outside the valid * range */ public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent, - final int dominantWinnerOverSecondRatio) { + final int dominantWinnerOverSecondRatio, final boolean contractUniformSubtrees) { this.reductionMode = Objects.requireNonNull(reductionMode, "reductionMode"); if (dominantWinnerMinPercent < 1 || dominantWinnerMinPercent > 100) { throw new IllegalArgumentException("dominantWinnerMinPercent must be in range 1..100."); @@ -80,6 +85,19 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM } this.dominantWinnerMinPercent = dominantWinnerMinPercent; this.dominantWinnerOverSecondRatio = dominantWinnerOverSecondRatio; + this.contractUniformSubtrees = contractUniformSubtrees; + } + + /** + * Creates a new instance without uniform-subtree contraction. + * + * @param reductionMode reduction mode + * @param dominantWinnerMinPercent minimum dominant winner percentage + * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio + */ + public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent, + final int dominantWinnerOverSecondRatio) { + this(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio, false); } /** @@ -93,4 +111,23 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM return new ReductionSettings(reductionMode, DEFAULT_DOMINANT_WINNER_MIN_PERCENT, DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO); } + + /** + * Returns settings that run uniform-subtree contraction before the configured + * subtree-merging mode. + * + *

    + * This is intended for Radixor patch-command tries, where a contracted accepting + * leaf can safely represent a subtree whose reachable entries all use the same + * patch command. + *

    + * + * @param settings base settings + * @return equivalent settings with uniform-subtree contraction enabled + */ + /* default */ static ReductionSettings withUniformSubtreeContraction(final ReductionSettings settings) { + Objects.requireNonNull(settings, "settings"); + return new ReductionSettings(settings.reductionMode(), settings.dominantWinnerMinPercent(), + settings.dominantWinnerOverSecondRatio(), true); + } } diff --git a/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java b/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java index 8d7e0cb..3da92b8 100644 --- a/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java +++ b/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java @@ -349,6 +349,7 @@ public final class StemmerKnowledgeExperiment { * @param trie compiled trie under test * @return immutable counts for this single input */ + @SuppressWarnings("deprecation") private static EvaluationCounts evaluateInput(final String input, final String expectedStem, final FrequencyTrie trie) { long getCorrect = 0L; diff --git a/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java b/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java index 5161a05..6b47b7f 100644 --- a/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java +++ b/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java @@ -39,6 +39,8 @@ import java.io.PushbackInputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; @@ -68,6 +70,7 @@ import java.util.zip.GZIPInputStream; * items containing Unicode whitespace characters while reporting them through * aggregated warning log records. */ +@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyMethods" }) public final class StemmerPatchTrieLoader { /* default */ static final String FILENAME_REQUIRED = "fileName required"; @@ -293,7 +296,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the dictionary cannot be found or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Language, boolean, ReductionSettings)} so + * patch commands are represented as {@link CompiledPatchCommand} + * values instead of reparsed {@link String} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Language language, final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { Objects.requireNonNull(language, "language"); @@ -303,6 +311,30 @@ public final class StemmerPatchTrieLoader { return load(language, storeOriginal, metadata); } + /** + * Loads a bundled dictionary and returns a runtime-specialized trie whose + * values are compiled patch commands. + * + *

    + * The text dictionary is still compiled through the canonical serialized + * patch-command representation. The returned trie replaces each stored + * serialized patch command with a {@link CompiledPatchCommand} so repeated + * runtime stemming does not parse patch-command strings. + *

    + * + * @param language bundled language dictionary + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionSettings reduction settings + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the dictionary cannot be found or read + */ + public static FrequencyTrie loadCompiled(final Language language, + final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { + return compilePatchTrie(load(language, storeOriginal, reductionSettings)); + } + /** * Loads a bundled dictionary using explicit trie compilation metadata. * @@ -320,7 +352,11 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the dictionary cannot be found or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Language, boolean, TrieMetadata)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Language language, final boolean storeOriginal, final TrieMetadata metadata) throws IOException { Objects.requireNonNull(language, "language"); @@ -335,6 +371,23 @@ public final class StemmerPatchTrieLoader { } } + /** + * Loads a bundled dictionary using explicit trie compilation metadata and + * returns a runtime-specialized trie whose values are compiled patch commands. + * + * @param language bundled language dictionary + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param metadata trie metadata describing the compilation configuration + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the dictionary cannot be found or read + */ + public static FrequencyTrie loadCompiled(final Language language, + final boolean storeOriginal, final TrieMetadata metadata) throws IOException { + return compilePatchTrie(load(language, storeOriginal, metadata)); + } + /** * Loads a bundled dictionary using default settings for the supplied reduction * mode. @@ -354,13 +407,35 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the dictionary cannot be found or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Language, boolean, ReductionMode)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Language language, final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { Objects.requireNonNull(reductionMode, "reductionMode"); return load(language, storeOriginal, ReductionSettings.withDefaults(reductionMode)); } + /** + * Loads a bundled dictionary using default settings for the supplied reduction + * mode and returns a runtime-specialized trie whose values are compiled patch + * commands. + * + * @param language bundled language dictionary + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionMode reduction mode + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the dictionary cannot be found or read + */ + public static FrequencyTrie loadCompiled(final Language language, + final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { + return compilePatchTrie(load(language, storeOriginal, reductionMode)); + } + /** * Loads a dictionary from a filesystem path using explicit reduction settings. * @@ -379,13 +454,35 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, ReductionSettings)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { return load(path, storeOriginal, reductionSettings, WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS); } + /** + * Loads a dictionary from a filesystem path using explicit reduction settings + * and returns a runtime-specialized trie whose values are compiled patch + * commands. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionSettings reduction settings + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { + return compilePatchTrie(load(path, storeOriginal, reductionSettings)); + } + /** * Loads a dictionary from a filesystem path using explicit reduction settings * and explicit traversal direction. @@ -405,7 +502,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection) throws IOException { @@ -413,6 +515,26 @@ public final class StemmerPatchTrieLoader { CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS); } + /** + * Loads a dictionary from a filesystem path using explicit reduction settings + * and traversal direction, returning runtime-specialized compiled patch values. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted using + * the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys and + * patch commands + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection) throws IOException { + return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection)); + } + /** * Loads a dictionary from a filesystem path using explicit reduction settings, * explicit traversal direction, and explicit case processing mode. @@ -432,7 +554,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) throws IOException { @@ -440,6 +567,29 @@ public final class StemmerPatchTrieLoader { DiacriticProcessingMode.AS_IS); } + /** + * Loads a dictionary from a filesystem path using explicit reduction settings, + * traversal direction, and case processing mode, returning runtime-specialized + * compiled patch values. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted using + * the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys and + * patch commands + * @param caseProcessingMode case processing mode used during dictionary parsing + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) + throws IOException { + return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode)); + } + /** * Loads a dictionary from a filesystem path using explicit reduction settings, * traversal direction, case processing mode, and diacritic processing mode. @@ -457,7 +607,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode, DiacriticProcessingMode)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode) @@ -468,6 +623,32 @@ public final class StemmerPatchTrieLoader { return load(path, storeOriginal, metadata); } + /** + * Loads a dictionary from a filesystem path using explicit semantic metadata + * dimensions, returning runtime-specialized compiled patch values. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted + * using the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys + * and patch commands + * @param caseProcessingMode case processing mode used during dictionary + * parsing + * @param diacriticProcessingMode diacritic processing mode used during + * dictionary parsing + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode, + final DiacriticProcessingMode diacriticProcessingMode) throws IOException { + return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode, + diacriticProcessingMode)); + } + /** * Loads a dictionary from a filesystem path using explicit trie compilation * metadata. @@ -485,7 +666,11 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, TrieMetadata)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final TrieMetadata metadata) throws IOException { Objects.requireNonNull(path, PARAMETER_PATH); @@ -498,6 +683,24 @@ public final class StemmerPatchTrieLoader { } } + /** + * Loads a dictionary from a filesystem path using explicit trie compilation + * metadata and returns a runtime-specialized trie whose values are compiled + * patch commands. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param metadata trie metadata describing the compilation configuration + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final TrieMetadata metadata) throws IOException { + return compilePatchTrie(load(path, storeOriginal, metadata)); + } + /** * Loads a dictionary from a filesystem path using default settings for the * supplied reduction mode. @@ -518,13 +721,35 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(Path, boolean, ReductionMode)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final Path path, final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { Objects.requireNonNull(reductionMode, "reductionMode"); return load(path, storeOriginal, ReductionSettings.withDefaults(reductionMode)); } + /** + * Loads a dictionary from a filesystem path using default settings for the + * supplied reduction mode and returns runtime-specialized compiled patch + * values. + * + * @param path path to the dictionary file + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionMode reduction mode + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final Path path, + final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { + return compilePatchTrie(load(path, storeOriginal, reductionMode)); + } + /** * Loads a dictionary from a filesystem path string using explicit reduction * settings. @@ -543,13 +768,36 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, ReductionSettings)} so + * patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { Objects.requireNonNull(fileName, FILENAME_REQUIRED); return load(Path.of(fileName), storeOriginal, reductionSettings); } + /** + * Loads a dictionary from a filesystem path string using explicit reduction + * settings and returns runtime-specialized compiled patch values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionSettings reduction settings + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings); + } + /** * Loads a dictionary from a filesystem path string using explicit reduction * settings and explicit traversal direction. @@ -571,7 +819,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection) throws IOException { @@ -580,6 +833,28 @@ public final class StemmerPatchTrieLoader { CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT); } + /** + * Loads a dictionary from a filesystem path string using explicit reduction + * settings and traversal direction, returning runtime-specialized compiled + * patch values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted using + * the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys and + * patch commands + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection) throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection); + } + /** * Loads a dictionary from a filesystem path string using explicit reduction * settings, explicit traversal direction, and explicit case processing mode. @@ -600,7 +875,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) throws IOException { @@ -609,6 +889,31 @@ public final class StemmerPatchTrieLoader { DiacriticProcessingMode.AS_IS); } + /** + * Loads a dictionary from a filesystem path string using explicit reduction + * settings, traversal direction, and case processing mode, returning + * runtime-specialized compiled patch values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted using + * the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys and + * patch commands + * @param caseProcessingMode case processing mode used during dictionary parsing + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) + throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, + caseProcessingMode); + } + /** * Loads a dictionary from a filesystem path string using explicit reduction * settings, explicit traversal direction, explicit case processing mode, and @@ -627,7 +932,12 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode, DiacriticProcessingMode)} + * so patch commands are represented as + * {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode) @@ -637,6 +947,33 @@ public final class StemmerPatchTrieLoader { diacriticProcessingMode); } + /** + * Loads a dictionary from a filesystem path string using explicit semantic + * metadata dimensions, returning runtime-specialized compiled patch values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted + * using the canonical no-op patch command + * @param reductionSettings reduction settings + * @param traversalDirection traversal direction used for both trie keys + * and patch commands + * @param caseProcessingMode case processing mode used during dictionary + * parsing + * @param diacriticProcessingMode diacritic processing mode used during + * dictionary parsing + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final ReductionSettings reductionSettings, + final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode, + final DiacriticProcessingMode diacriticProcessingMode) throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, + caseProcessingMode, diacriticProcessingMode); + } + /** * Loads a dictionary from a filesystem path string using explicit trie * compilation metadata. @@ -652,13 +989,35 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, TrieMetadata)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final TrieMetadata metadata) throws IOException { Objects.requireNonNull(fileName, FILENAME_REQUIRED); return load(Path.of(fileName), storeOriginal, metadata); } + /** + * Loads a dictionary from a filesystem path string using explicit trie + * compilation metadata and returns runtime-specialized compiled patch values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param metadata trie metadata describing the compilation configuration + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final TrieMetadata metadata) throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, metadata); + } + /** * Loads a dictionary from a filesystem path string using default settings for * the supplied reduction mode. @@ -677,13 +1036,36 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if any argument is {@code null} * @throws IOException if the file cannot be opened or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadCompiled(String, boolean, ReductionMode)} so patch + * commands are represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie load(final String fileName, final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { Objects.requireNonNull(fileName, FILENAME_REQUIRED); return load(Path.of(fileName), storeOriginal, reductionMode); } + /** + * Loads a dictionary from a filesystem path string using default settings for + * the supplied reduction mode and returns runtime-specialized compiled patch + * values. + * + * @param fileName file name or path string + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionMode reduction mode + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the file cannot be opened or read + */ + public static FrequencyTrie loadCompiled(final String fileName, + final boolean storeOriginal, final ReductionMode reductionMode) throws IOException { + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return loadCompiled(Path.of(fileName), storeOriginal, reductionMode); + } + /** * Parses one dictionary and builds the compiled trie. * @@ -736,7 +1118,9 @@ public final class StemmerPatchTrieLoader { Objects.requireNonNull(reductionSettings, "reductionSettings"); Objects.requireNonNull(caseProcessingMode, "caseProcessingMode"); Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode"); - return TrieMetadata.forCompilation(traversalDirection, reductionSettings, diacriticProcessingMode, + final ReductionSettings patchReductionSettings = ReductionSettings + .withUniformSubtreeContraction(reductionSettings); + return TrieMetadata.forCompilation(traversalDirection, patchReductionSettings, diacriticProcessingMode, caseProcessingMode); } @@ -750,6 +1134,28 @@ public final class StemmerPatchTrieLoader { return language.isRightToLeft() ? WordTraversalDirection.FORWARD : WordTraversalDirection.BACKWARD; } + /** + * Maps textual patch commands to runtime-specialized compiled patch commands. + * + *

    + * Equal textual patch commands are compiled once and shared by all trie values + * that reference them. The returned trie preserves the source trie keys, + * metadata, traversal direction, counts, and reduction settings. + *

    + * + * @param trie source trie containing textual patch commands + * @return equivalent trie containing compiled patch commands + * @throws NullPointerException if {@code trie} is {@code null} + */ + private static FrequencyTrie compilePatchTrie(final FrequencyTrie trie) { + final FrequencyTrie sourceTrie = Objects.requireNonNull(trie, "trie"); + final Map compiledPatches = new HashMap<>(4096); + return FrequencyTrieBuilders.mapValues(sourceTrie, CompiledPatchCommand[]::new, + sourceTrie.metadata().reductionSettings(), + patch -> compiledPatches.computeIfAbsent(patch, + value -> CompiledPatchCommand.compile(value, sourceTrie.traversalDirection()))); + } + /** * Loads a GZip-compressed binary patch-command trie from a filesystem path. * @@ -758,12 +1164,30 @@ public final class StemmerPatchTrieLoader { * @throws NullPointerException if {@code path} is {@code null} * @throws IOException if the file cannot be opened, decompressed, or * read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadBinaryCompiled(Path)} so patch commands are + * represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie loadBinary(final Path path) throws IOException { Objects.requireNonNull(path, PARAMETER_PATH); return StemmerPatchTrieBinaryIO.read(path); } + /** + * Loads a GZip-compressed binary patch-command trie from a filesystem path and + * returns runtime-specialized compiled patch values. + * + * @param path path to the compressed binary trie file + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if {@code path} is {@code null} + * @throws IOException if the file cannot be opened, decompressed, or + * read + */ + public static FrequencyTrie loadBinaryCompiled(final Path path) throws IOException { + return compilePatchTrie(loadBinary(path)); + } + /** * Loads a GZip-compressed binary patch-command trie from a filesystem path * using a custom dense lookup span override. @@ -779,12 +1203,34 @@ public final class StemmerPatchTrieLoader { * @throws NullPointerException if {@code path} is {@code null} * @throws IOException if the file cannot be opened, decompressed, or * read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadBinaryCompiled(Path, int)} so patch commands are + * represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie loadBinary(final Path path, final int maxExpandedIndex) throws IOException { Objects.requireNonNull(path, PARAMETER_PATH); return StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex); } + /** + * Loads a GZip-compressed binary patch-command trie from a filesystem path using + * a custom dense lookup span override and returns runtime-specialized compiled + * patch values. + * + * @param path path to the compressed binary trie file + * @param maxExpandedIndex dense lookup span override; negative values use + * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if {@code path} is {@code null} + * @throws IOException if the file cannot be opened, decompressed, or + * read + */ + public static FrequencyTrie loadBinaryCompiled(final Path path, + final int maxExpandedIndex) throws IOException { + return compilePatchTrie(loadBinary(path, maxExpandedIndex)); + } + /** * Loads a GZip-compressed binary patch-command trie from a filesystem path * string. @@ -794,12 +1240,30 @@ public final class StemmerPatchTrieLoader { * @throws NullPointerException if {@code fileName} is {@code null} * @throws IOException if the file cannot be opened, decompressed, or * read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadBinaryCompiled(String)} so patch commands are + * represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie loadBinary(final String fileName) throws IOException { Objects.requireNonNull(fileName, FILENAME_REQUIRED); return StemmerPatchTrieBinaryIO.read(fileName); } + /** + * Loads a GZip-compressed binary patch-command trie from a filesystem path + * string and returns runtime-specialized compiled patch values. + * + * @param fileName file name or path string + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if {@code fileName} is {@code null} + * @throws IOException if the file cannot be opened, decompressed, or + * read + */ + public static FrequencyTrie loadBinaryCompiled(final String fileName) throws IOException { + return compilePatchTrie(loadBinary(fileName)); + } + /** * Loads a GZip-compressed binary patch-command trie from a filesystem path * string using a custom dense lookup span override. @@ -815,13 +1279,35 @@ public final class StemmerPatchTrieLoader { * @throws NullPointerException if {@code fileName} is {@code null} * @throws IOException if the file cannot be opened, decompressed, or * read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadBinaryCompiled(String, int)} so patch commands are + * represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie loadBinary(final String fileName, final int maxExpandedIndex) throws IOException { Objects.requireNonNull(fileName, FILENAME_REQUIRED); return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex); } + /** + * Loads a GZip-compressed binary patch-command trie from a filesystem path string + * using a custom dense lookup span override and returns runtime-specialized + * compiled patch values. + * + * @param fileName file name or path string + * @param maxExpandedIndex dense lookup span override; negative values use + * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if {@code fileName} is {@code null} + * @throws IOException if the file cannot be opened, decompressed, or + * read + */ + public static FrequencyTrie loadBinaryCompiled(final String fileName, + final int maxExpandedIndex) throws IOException { + return compilePatchTrie(loadBinary(fileName, maxExpandedIndex)); + } + /** * Loads a GZip-compressed binary patch-command trie from an input stream. * @@ -829,12 +1315,30 @@ public final class StemmerPatchTrieLoader { * @return compiled patch-command trie * @throws NullPointerException if {@code inputStream} is {@code null} * @throws IOException if the stream cannot be decompressed or read + * @deprecated Since 2.3.0 for runtime stemming. Use + * {@link #loadBinaryCompiled(InputStream)} so patch commands are + * represented as {@link CompiledPatchCommand} values. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static FrequencyTrie loadBinary(final InputStream inputStream) throws IOException { Objects.requireNonNull(inputStream, "inputStream"); return StemmerPatchTrieBinaryIO.read(inputStream); } + /** + * Loads a GZip-compressed binary patch-command trie from an input stream and + * returns runtime-specialized compiled patch values. + * + * @param inputStream source input stream + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if {@code inputStream} is {@code null} + * @throws IOException if the stream cannot be decompressed or read + */ + public static FrequencyTrie loadBinaryCompiled(final InputStream inputStream) + throws IOException { + return compilePatchTrie(loadBinary(inputStream)); + } + /** * Loads only persisted metadata from a GZip-compressed binary patch-command * trie file. diff --git a/src/main/java/org/egothor/stemmer/TrieMetadata.java b/src/main/java/org/egothor/stemmer/TrieMetadata.java index 02c2759..d7a752a 100644 --- a/src/main/java/org/egothor/stemmer/TrieMetadata.java +++ b/src/main/java/org/egothor/stemmer/TrieMetadata.java @@ -167,6 +167,8 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi .append("dominantWinnerOverSecondRatio=").append(this.reductionSettings.dominantWinnerOverSecondRatio()) .append('\n') // + .append("contractUniformSubtrees=").append(this.reductionSettings.contractUniformSubtrees()).append('\n') + // .append("diacriticProcessingMode=").append(this.diacriticProcessingMode.name()).append('\n') // .append("caseProcessingMode=").append(this.caseProcessingMode.name()).append('\n'); @@ -207,13 +209,16 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi final int dominantWinnerMinPercent = Integer.parseInt(requireEntry(entries, "dominantWinnerMinPercent")); final int dominantWinnerOverSecondRatio = Integer // NOPMD .parseInt(requireEntry(entries, "dominantWinnerOverSecondRatio")); + final boolean contractUniformSubtrees = Boolean + .parseBoolean(entries.getOrDefault("contractUniformSubtrees", "false")); final DiacriticProcessingMode diacriticProcessingMode = DiacriticProcessingMode .valueOf(requireEntry(entries, "diacriticProcessingMode")); final CaseProcessingMode caseProcessingMode = CaseProcessingMode .valueOf(requireEntry(entries, "caseProcessingMode")); return new TrieMetadata(formatVersion, traversalDirection, - new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio), + new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio, + contractUniformSubtrees), diacriticProcessingMode, caseProcessingMode); } diff --git a/src/main/java/org/egothor/stemmer/trie/CompiledNode.java b/src/main/java/org/egothor/stemmer/trie/CompiledNode.java index 8d527f7..74cb661 100644 --- a/src/main/java/org/egothor/stemmer/trie/CompiledNode.java +++ b/src/main/java/org/egothor/stemmer/trie/CompiledNode.java @@ -93,6 +93,11 @@ public final class CompiledNode { */ private final int[] orderedCounts; + /** + * Whether this node accepts any remaining lookup input. + */ + private final boolean acceptsRemainingInput; + /** * Creates one validated compiled node using {@link #DEFAULT_MAX_EXPANDED_INDEX} * for dense lookup sizing. @@ -120,6 +125,23 @@ public final class CompiledNode { */ public CompiledNode(final char[] edgeLabels, final CompiledNode[] children, final V[] orderedValues, final int maxExpandedIndex, final int... orderedCounts) { + this(edgeLabels, children, orderedValues, false, maxExpandedIndex, orderedCounts); + } + + /** + * Creates one validated compiled node. + * + * @param acceptsRemainingInput whether this node accepts any remaining lookup + * input + * @param maxExpandedIndex upper bound for the dense lookup interval size + * @throws NullPointerException if any array argument is {@code null} + * @throws IllegalArgumentException if the edge-related arrays or value-related + * arrays do not have matching lengths, the + * dense interval size is negative, or an + * accepting node has children + */ + public CompiledNode(final char[] edgeLabels, final CompiledNode[] children, final V[] orderedValues, + final boolean acceptsRemainingInput, final int maxExpandedIndex, final int... orderedCounts) { Objects.requireNonNull(edgeLabels, "edgeLabels"); Objects.requireNonNull(children, "children"); Objects.requireNonNull(orderedValues, "orderedValues"); @@ -135,11 +157,18 @@ public final class CompiledNode { if (orderedValues.length != orderedCounts.length) { throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length."); } + if (acceptsRemainingInput && edgeLabels.length != 0) { + throw new IllegalArgumentException("Accepting nodes cannot have child edges."); + } + if (acceptsRemainingInput && orderedValues.length == 0) { + throw new IllegalArgumentException("Accepting nodes must store at least one value."); + } this.edgeLabels = edgeLabels; this.children = children; this.orderedValues = orderedValues; this.orderedCounts = orderedCounts; + this.acceptsRemainingInput = acceptsRemainingInput; if (edgeLabels.length == 0 || maxExpandedIndex == 0) { this.denseChildren = null; @@ -268,6 +297,15 @@ public final class CompiledNode { return !hasChildren(); } + /** + * Indicates whether this node accepts any remaining lookup input. + * + * @return {@code true} for a contracted accepting leaf + */ + public boolean acceptsRemainingInput() { + return this.acceptsRemainingInput; + } + /** * Tests whether an edge label is present at this node. * @@ -310,6 +348,7 @@ public final class CompiledNode { hash = 31 * hash + Arrays.hashCode(this.orderedValues); hash = 31 * hash + Arrays.hashCode(this.orderedCounts); hash = 31 * hash + Objects.hash(this.denseEdgeMin); + hash = 31 * hash + Boolean.hashCode(this.acceptsRemainingInput); hash = 31 * hash + (hasDenseLookup() ? Arrays.hashCode(this.denseChildren) : 0); return hash; } @@ -331,6 +370,7 @@ public final class CompiledNode { return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children) && Arrays.equals(this.orderedValues, other.orderedValues) && Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin + && this.acceptsRemainingInput == other.acceptsRemainingInput && Arrays.equals(this.denseChildren, other.denseChildren); } @@ -342,7 +382,8 @@ public final class CompiledNode { @Override public String toString() { return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount=" - + this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}'; + + this.orderedValues.length + ", acceptsRemainingInput=" + this.acceptsRemainingInput + + ", denseTableLength=" + denseTableLength() + '}'; } /** diff --git a/src/main/java/org/egothor/stemmer/trie/ReducedNode.java b/src/main/java/org/egothor/stemmer/trie/ReducedNode.java index 2430a9d..6f40db5 100644 --- a/src/main/java/org/egothor/stemmer/trie/ReducedNode.java +++ b/src/main/java/org/egothor/stemmer/trie/ReducedNode.java @@ -60,18 +60,38 @@ public final class ReducedNode { */ private final Map> children; + /** + * Whether this reduced node accepts any remaining lookup input. + */ + private final boolean acceptsRemainingInput; + /** * Creates a new reduced node. * * @param signature reduction signature * @param localCounts local counts * @param children children + * @param acceptsRemainingInput whether this node accepts any remaining lookup + * input + */ + public ReducedNode(final ReductionSignature signature, final Map localCounts, + final Map> children, final boolean acceptsRemainingInput) { + this.signature = signature; + this.localCounts = new LinkedHashMap<>(localCounts); + this.children = new LinkedHashMap<>(children); + this.acceptsRemainingInput = acceptsRemainingInput; + } + + /** + * Creates a new non-accepting reduced node. + * + * @param signature reduction signature + * @param localCounts local counts + * @param children children */ public ReducedNode(final ReductionSignature signature, final Map localCounts, final Map> children) { - this.signature = signature; - this.localCounts = new LinkedHashMap<>(localCounts); - this.children = new LinkedHashMap<>(children); + this(signature, localCounts, children, false); } /** @@ -111,6 +131,15 @@ public final class ReducedNode { return this.children; } + /** + * Returns whether this node accepts any remaining lookup input. + * + * @return {@code true} for a contracted accepting leaf + */ + public boolean acceptsRemainingInput() { + return this.acceptsRemainingInput; + } + /** * Merges additional local counts into this node. * diff --git a/src/main/java/org/egothor/stemmer/trie/ReductionSignature.java b/src/main/java/org/egothor/stemmer/trie/ReductionSignature.java index 3518c01..bdf6b8a 100644 --- a/src/main/java/org/egothor/stemmer/trie/ReductionSignature.java +++ b/src/main/java/org/egothor/stemmer/trie/ReductionSignature.java @@ -55,15 +55,22 @@ public final class ReductionSignature { */ private final List> childDescriptors; + /** + * Whether the represented node accepts any remaining lookup input. + */ + private final boolean acceptsRemainingInput; + /** * Creates a signature. * * @param localDescriptor local descriptor * @param childDescriptors child descriptors */ - private ReductionSignature(final Object localDescriptor, final List> childDescriptors) { + private ReductionSignature(final Object localDescriptor, final List> childDescriptors, + final boolean acceptsRemainingInput) { this.localDescriptor = localDescriptor; this.childDescriptors = childDescriptors; + this.acceptsRemainingInput = acceptsRemainingInput; } /** @@ -72,11 +79,14 @@ public final class ReductionSignature { * @param localSummary local value summary * @param children reduced children * @param settings reduction settings + * @param acceptsRemainingInput whether this node accepts any remaining lookup + * input * @param value type * @return subtree signature */ public static ReductionSignature create(final LocalValueSummary localSummary, - final Map> children, final ReductionSettings settings) { + final Map> children, final ReductionSettings settings, + final boolean acceptsRemainingInput) { final Object localDescriptor = switch (settings.reductionMode()) { case MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS -> RankedLocalDescriptor.of(localSummary.orderedValues()); @@ -100,12 +110,28 @@ public final class ReductionSignature { childDescriptors.add(new ChildDescriptor<>(entry.getKey(), entry.getValue().signature())); } - return new ReductionSignature<>(localDescriptor, Collections.unmodifiableList(childDescriptors)); + return new ReductionSignature<>(localDescriptor, Collections.unmodifiableList(childDescriptors), + acceptsRemainingInput); + } + + /** + * Creates a non-accepting subtree signature according to the selected reduction + * mode. + * + * @param localSummary local value summary + * @param children reduced children + * @param settings reduction settings + * @param value type + * @return subtree signature + */ + public static ReductionSignature create(final LocalValueSummary localSummary, + final Map> children, final ReductionSettings settings) { + return create(localSummary, children, settings, false); } @Override public int hashCode() { - return Objects.hash(this.localDescriptor, this.childDescriptors); + return Objects.hash(this.localDescriptor, this.childDescriptors, this.acceptsRemainingInput); } @Override @@ -118,6 +144,7 @@ public final class ReductionSignature { } final ReductionSignature that = (ReductionSignature) other; return Objects.equals(this.localDescriptor, that.localDescriptor) - && Objects.equals(this.childDescriptors, that.childDescriptors); + && Objects.equals(this.childDescriptors, that.childDescriptors) + && this.acceptsRemainingInput == that.acceptsRemainingInput; } } diff --git a/src/test/java/org/egothor/stemmer/CompileIntegrationTest.java b/src/test/java/org/egothor/stemmer/CompileIntegrationTest.java index f138261..94109da 100644 --- a/src/test/java/org/egothor/stemmer/CompileIntegrationTest.java +++ b/src/test/java/org/egothor/stemmer/CompileIntegrationTest.java @@ -100,6 +100,7 @@ import org.junit.jupiter.params.provider.MethodSource; @Tag("slow") @TestInstance(TestInstance.Lifecycle.PER_CLASS) @DisplayName("Compile integration") +@SuppressWarnings("deprecation") final class CompileIntegrationTest { /** diff --git a/src/test/java/org/egothor/stemmer/CompiledPatchCommandTest.java b/src/test/java/org/egothor/stemmer/CompiledPatchCommandTest.java new file mode 100644 index 0000000..6de7f93 --- /dev/null +++ b/src/test/java/org/egothor/stemmer/CompiledPatchCommandTest.java @@ -0,0 +1,341 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Unit tests for {@link CompiledPatchCommand}. + */ +@DisplayName("CompiledPatchCommand") +@Tag("unit") +@Tag("stemmer") +@Tag("patch") +@SuppressWarnings("deprecation") +final class CompiledPatchCommandTest { + + /** + * Provides representative source-target pairs for compiled command validation. + * + * @return test arguments + */ + private static Stream provideRoundTripPairs() { + return Stream.of( + Arguments.of(WordTraversalDirection.BACKWARD, "", ""), + Arguments.of(WordTraversalDirection.BACKWARD, "a", "a"), + Arguments.of(WordTraversalDirection.BACKWARD, "a", "b"), + Arguments.of(WordTraversalDirection.BACKWARD, "abc", "ab"), + Arguments.of(WordTraversalDirection.BACKWARD, "ab", "abc"), + Arguments.of(WordTraversalDirection.BACKWARD, "teacher", "teach"), + Arguments.of(WordTraversalDirection.BACKWARD, "running", "run"), + Arguments.of(WordTraversalDirection.BACKWARD, "cities", "city"), + Arguments.of(WordTraversalDirection.BACKWARD, "mississippi", "missouri"), + Arguments.of(WordTraversalDirection.FORWARD, "", ""), + Arguments.of(WordTraversalDirection.FORWARD, "a", "a"), + Arguments.of(WordTraversalDirection.FORWARD, "a", "b"), + Arguments.of(WordTraversalDirection.FORWARD, "abc", "bc"), + Arguments.of(WordTraversalDirection.FORWARD, "bc", "abc"), + Arguments.of(WordTraversalDirection.FORWARD, "transformation", "transform"), + Arguments.of(WordTraversalDirection.FORWARD, "cities", "city")); + } + + /** + * Provides malformed compatibility patch commands. + * + * @return test arguments + */ + private static Stream providePreservePatchCommands() { + return Stream.of( + Arguments.of((Object) null), + Arguments.of(""), + Arguments.of("D`"), + Arguments.of("-`"), + Arguments.of("DaX")); + } + + /** + * Provides compound patch commands that stress direct compiled execution. + * + * @return test arguments + */ + private static Stream provideCompoundPatchCommands() { + return Stream.of( + Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "IaIbIc"), + Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-bDcIxRy"), + Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "DbIxIy-cRz"), + Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-z"), + Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-zIx"), + Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "IxIyIz"), + Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-bDcIxRy"), + Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "DbIxIy-cRz"), + Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-z"), + Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-zIx"), + Arguments.of(WordTraversalDirection.BACKWARD, "a", "DaDa")); + } + + /** + * Provides patch commands whose length delta would produce an empty stem. + * + * @return test arguments + */ + private static Stream provideEmptyStemPatchCommands() { + return Stream.of( + Arguments.of(WordTraversalDirection.BACKWARD, "a", "Da"), + Arguments.of(WordTraversalDirection.FORWARD, "a", "Da"), + Arguments.of(WordTraversalDirection.FORWARD, "a", "DaDa")); + } + + /** + * Verifies that representative serialized commands compile to concrete command + * classes instead of one universal runtime-dispatched command shape. + */ + @Test + @DisplayName("compiles representative commands to concrete command classes") + void shouldCompileRepresentativeCommandsToConcreteClasses() { + assertAll( + () -> assertEquals("DeleteSuffixCommand", + CompiledPatchCommand.compile("Da", WordTraversalDirection.BACKWARD) + .getClass().getSimpleName()), + () -> assertEquals("DeletePrefixCommand", + CompiledPatchCommand.compile("Da", WordTraversalDirection.FORWARD) + .getClass().getSimpleName()), + () -> assertEquals("AppendCharacterCommand", + CompiledPatchCommand.compile("Ix", WordTraversalDirection.BACKWARD) + .getClass().getSimpleName()), + () -> assertEquals("PrependCharacterCommand", + CompiledPatchCommand.compile("Ix", WordTraversalDirection.FORWARD) + .getClass().getSimpleName()), + () -> assertEquals("ReplaceLastCharacterCommand", + CompiledPatchCommand.compile("Rx", WordTraversalDirection.BACKWARD) + .getClass().getSimpleName()), + () -> assertEquals("ReplaceFirstCharacterCommand", + CompiledPatchCommand.compile("Rx", WordTraversalDirection.FORWARD) + .getClass().getSimpleName()), + () -> assertEquals("BackwardCompoundCommand", + CompiledPatchCommand.compile("-aDa", WordTraversalDirection.BACKWARD) + .getClass().getSimpleName()), + () -> assertEquals("ForwardCompoundCommand", + CompiledPatchCommand.compile("-aDa", WordTraversalDirection.FORWARD) + .getClass().getSimpleName())); + } + + /** + * Verifies the no-op fast-path marker used by high-throughput callers. + */ + @Test + @DisplayName("marks only all-source preserve commands as preserving") + void shouldMarkOnlyAllSourcePreserveCommandsAsPreserving() { + assertAll( + () -> assertEquals(true, + CompiledPatchCommand.compile(null, WordTraversalDirection.BACKWARD).preservesAllSources()), + () -> assertEquals(true, + CompiledPatchCommand.compile("", WordTraversalDirection.BACKWARD).preservesAllSources()), + () -> assertEquals(true, + CompiledPatchCommand.compile("Na", WordTraversalDirection.BACKWARD).preservesAllSources()), + () -> assertEquals(true, + CompiledPatchCommand.compile("-a", WordTraversalDirection.BACKWARD).preservesAllSources()), + () -> assertEquals(false, + CompiledPatchCommand.compile("Da", WordTraversalDirection.BACKWARD).preservesAllSources()), + () -> assertEquals(false, + CompiledPatchCommand.compile("Ix", WordTraversalDirection.FORWARD).preservesAllSources()), + () -> assertEquals(false, + CompiledPatchCommand.compile("-aDa", WordTraversalDirection.BACKWARD) + .preservesAllSources())); + } + + /** + * Verifies that compiled commands match the string interpreter. + * + * @param traversalDirection traversal direction + * @param source source word + * @param target target word + */ + @ParameterizedTest + @MethodSource("provideRoundTripPairs") + @DisplayName("matches interpreted patch application") + void shouldMatchInterpretedPatchApplication(final WordTraversalDirection traversalDirection, final String source, + final String target) { + final PatchCommandEncoder encoder = PatchCommandEncoder.builder() + .traversalDirection(traversalDirection) + .build(); + final String patch = encoder.encode(source, target); + final CompiledPatchCommand compiled = encoder.compile(patch); + final String expected = PatchCommandEncoder.apply(source, patch, traversalDirection); + + final char[] sequenceOutput = new char[Math.max(source.length(), expected.length()) + 8]; + final int sequenceLength = compiled.applyTo(source, sequenceOutput, 2, sequenceOutput.length - 2); + + final char[] sourceArray = source.toCharArray(); + final char[] arrayOutput = new char[Math.max(source.length(), expected.length()) + 8]; + final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 1, + arrayOutput.length - 1); + + assertAll( + () -> assertEquals(expected, compiled.apply(source)), + () -> assertEquals(expected.length(), sequenceLength), + () -> assertEquals(expected, new String(sequenceOutput, 2, sequenceLength)), + () -> assertEquals(expected.length(), arrayLength), + () -> assertEquals(expected, new String(arrayOutput, 1, arrayLength))); + } + + /** + * Verifies compound direct execution against the compatibility interpreter. + * + * @param traversalDirection traversal direction + * @param source source word + * @param patch serialized compound patch command + */ + @ParameterizedTest + @MethodSource("provideCompoundPatchCommands") + @DisplayName("matches interpreted compound patch application") + void shouldMatchInterpretedCompoundPatchApplication(final WordTraversalDirection traversalDirection, + final String source, final String patch) { + final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patch, traversalDirection); + final String expected = PatchCommandEncoder.apply(source, patch, traversalDirection); + final char[] sequenceOutput = new char[Math.max(source.length(), expected.length()) + 8]; + final char[] sourceArray = source.toCharArray(); + final char[] arrayOutput = new char[Math.max(source.length(), expected.length()) + 8]; + + final int sequenceLength = compiled.applyTo(source, sequenceOutput, 2, sequenceOutput.length - 2); + final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 1, + arrayOutput.length - 1); + + assertAll( + () -> assertEquals(expected, compiled.apply(source)), + () -> assertEquals(expected.length(), sequenceLength), + () -> assertEquals(expected, new String(sequenceOutput, 2, sequenceLength)), + () -> assertEquals(expected.length(), arrayLength), + () -> assertEquals(expected, new String(arrayOutput, 1, arrayLength))); + } + + /** + * Verifies that the compiled hot path never produces an empty stem. + * + * @param traversalDirection traversal direction + * @param source source word + * @param patch serialized patch command + */ + @ParameterizedTest + @MethodSource("provideEmptyStemPatchCommands") + @DisplayName("preserves source when a patch would produce an empty stem") + void shouldPreserveSourceWhenPatchWouldProduceEmptyStem(final WordTraversalDirection traversalDirection, + final String source, final String patch) { + final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patch, traversalDirection); + final char[] sequenceOutput = new char[source.length() + 4]; + final char[] sourceArray = source.toCharArray(); + final char[] arrayOutput = new char[source.length() + 4]; + + final int sequenceLength = compiled.applyTo(source, sequenceOutput, 1, sequenceOutput.length - 1); + final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 2, + arrayOutput.length - 2); + + assertAll( + () -> assertSame(source, compiled.apply(source)), + () -> assertEquals(source.length(), sequenceLength), + () -> assertEquals(source, new String(sequenceOutput, 1, sequenceLength)), + () -> assertEquals(source.length(), arrayLength), + () -> assertEquals(source, new String(arrayOutput, 2, arrayLength))); + } + + /** + * Verifies preserve-only patch commands. + * + * @param patchCommand serialized patch command + */ + @ParameterizedTest + @MethodSource("providePreservePatchCommands") + @DisplayName("preserves source for interpreted preserve-only commands") + void shouldPreserveSourceForPreserveOnlyCommands(final String patchCommand) { + final String source = "teacher"; + final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patchCommand, WordTraversalDirection.BACKWARD); + + assertSame(source, compiled.apply(source)); + } + + /** + * Verifies insufficient output capacity reporting. + */ + @ParameterizedTest + @MethodSource("provideRoundTripPairs") + @DisplayName("reports insufficient output capacity") + void shouldReportInsufficientOutputCapacity(final WordTraversalDirection traversalDirection, final String source, + final String target) { + final PatchCommandEncoder encoder = PatchCommandEncoder.builder() + .traversalDirection(traversalDirection) + .build(); + final String patch = encoder.encode(source, target); + final CompiledPatchCommand compiled = encoder.compile(patch); + final String expected = compiled.apply(source); + final char[] output = new char[Math.max(0, expected.length() - 1)]; + + if (expected.isEmpty()) { + assertEquals(0, compiled.applyTo(source, output, 0, output.length)); + } else { + assertEquals(CompiledPatchCommand.APPLY_INSUFFICIENT_CAPACITY, + compiled.applyTo(source, output, 0, output.length)); + } + } + + /** + * Verifies compile-time rejection of unsupported serialized commands. + */ + @ParameterizedTest + @MethodSource("provideInvalidPatchCommands") + @DisplayName("rejects unsupported patch commands") + void shouldRejectUnsupportedPatchCommands(final String patchCommand) { + assertThrows(IllegalArgumentException.class, + () -> CompiledPatchCommand.compile(patchCommand, WordTraversalDirection.BACKWARD)); + } + + /** + * Provides invalid patch commands. + * + * @return test arguments + */ + private static Stream provideInvalidPatchCommands() { + return Stream.of( + Arguments.of("Xa"), + Arguments.of("N`"), + Arguments.of("DaN`")); + } +} diff --git a/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java b/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java index 271f568..a5b6945 100644 --- a/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java +++ b/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java @@ -76,6 +76,7 @@ import org.junit.jupiter.params.provider.MethodSource; @Tag("serialization") @Tag("trie") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SuppressWarnings("deprecation") final class CompiledTrieArtifactRegressionTest { /** diff --git a/src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java b/src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java index c850346..9fec15e 100644 --- a/src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java +++ b/src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java @@ -269,6 +269,25 @@ class FrequencyTrieBuildersTest { assertTrieStateEquals(original, reconstructed, "xy"); } + /** + * Verifies that compiled trie values can be mapped to another value type while + * preserving lookup semantics and local counts. + */ + @Test + @DisplayName("should map values while preserving keys and counts") + void shouldMapValuesWhilePreservingKeysAndCounts() { + final FrequencyTrie original = createRepresentativeTrie(); + + final FrequencyTrie mapped = FrequencyTrieBuilders.mapValues(original, ARRAY_FACTORY, + RANKED_SETTINGS, value -> "mapped-" + value); + + assertAll( + () -> assertEquals("mapped-root-main", mapped.get("")), + () -> assertArrayEquals(new String[] { "mapped-A1", "mapped-A2" }, mapped.getAll("a")), + () -> assertIterableEquals(List.of(new ValueCount("mapped-AB1", 5), + new ValueCount("mapped-AB2", 2)), mapped.getEntries("ab"))); + } + /** * Verifies the documented null-argument contract for both public reconstruction * entry points. @@ -292,7 +311,19 @@ class FrequencyTrieBuildersTest { () -> FrequencyTrieBuilders.copyOf(trie, null, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)), () -> assertThrows(NullPointerException.class, - () -> FrequencyTrieBuilders.copyOf(trie, ARRAY_FACTORY, (ReductionMode) null))); + () -> FrequencyTrieBuilders.copyOf(trie, ARRAY_FACTORY, (ReductionMode) null)), + () -> assertThrows(NullPointerException.class, + () -> FrequencyTrieBuilders.mapValues(null, Integer[]::new, RANKED_SETTINGS, String::length)), + () -> assertThrows(NullPointerException.class, + () -> FrequencyTrieBuilders.mapValues(trie, null, RANKED_SETTINGS, String::length)), + () -> assertThrows(NullPointerException.class, + () -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, (ReductionSettings) null, + String::length)), + () -> assertThrows(NullPointerException.class, + () -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, RANKED_SETTINGS, null)), + () -> assertThrows(NullPointerException.class, + () -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, (ReductionMode) null, + String::length))); } /** diff --git a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java index 67220f2..09ea9bf 100644 --- a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java +++ b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java @@ -94,6 +94,17 @@ class FrequencyTrieTest { ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); } + /** + * Creates reduction settings with the internal uniform-subtree contraction + * enabled. + * + * @return contraction-enabled settings + */ + private static ReductionSettings uniformSubtreeContractionSettings() { + return ReductionSettings.withUniformSubtreeContraction(ReductionSettings + .withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + } + /** * Verifies that the builder rejects {@code null} constructor arguments. */ @@ -508,6 +519,10 @@ class FrequencyTrieTest { () -> trie.getAllNormalized((CharSequence) null, sink, 1)), () -> assertThrows(NullPointerException.class, () -> trie.getAllNormalized("house", null, 1)), + () -> assertThrows(NullPointerException.class, + () -> trie.getNormalized(null)), + () -> assertThrows(NullPointerException.class, + () -> trie.getNormalizedString(null)), () -> assertThrows(NullPointerException.class, () -> trie.getAll((CharSequence) null, sink, 1)), () -> assertThrows(NullPointerException.class, @@ -537,6 +552,12 @@ class FrequencyTrieTest { }, 10)), () -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true), "Normalized lookup must bypass metadata lowercasing."), + () -> assertNull(trie.getNormalized("HOUSE"), + "Normalized preferred lookup must bypass metadata lowercasing."), + () -> assertNull(trie.getNormalizedString("HOUSE"), + "String-specialized normalized lookup must bypass metadata lowercasing."), + () -> assertEquals("noun", trie.getNormalized("house")), + () -> assertEquals("noun", trie.getNormalizedString("house")), () -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> { assertEquals("noun", value); return true; @@ -1056,6 +1077,103 @@ class FrequencyTrieTest { () -> assertEquals(original.get("z"), disabledDense.get("z"))); } + /** + * Verifies that uniform subtree contraction is not part of the default generic + * trie semantics. + */ + @Test + @Tag("reduction") + @DisplayName("Default reduction keeps exact lookup semantics for uniform subtrees") + void shouldKeepExactLookupWhenUniformSubtreeContractionIsDisabled() { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder<>(String[]::new, + ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS), + WordTraversalDirection.FORWARD); + builder.put("aa", "x"); + builder.put("ab", "x"); + + final FrequencyTrie trie = builder.build(); + + assertAll("exact lookup", + () -> assertEquals("x", trie.get("aa")), + () -> assertEquals("x", trie.get("ab")), + () -> assertNull(trie.get("az")), + () -> assertFalse(trie.root().findChild('a').acceptsRemainingInput())); + } + + /** + * Verifies that the internal uniform-subtree contraction replaces a uniform + * non-leaf subtree with an accepting leaf. + */ + @Test + @Tag("reduction") + @DisplayName("Uniform subtree contraction replaces uniform internal subtree with accepting leaf") + void shouldContractUniformInternalSubtreeIntoAcceptingLeaf() { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder<>(String[]::new, + uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD); + builder.put("aa", "x"); + builder.put("ab", "x"); + builder.put("ba", "y"); + + final FrequencyTrie trie = builder.build(); + + assertAll("contracted lookup", + () -> assertEquals(3, trie.size()), + () -> assertTrue(trie.root().findChild('a').acceptsRemainingInput()), + () -> assertEquals("x", trie.get("a")), + () -> assertEquals("x", trie.get("aa")), + () -> assertEquals("x", trie.get("ab")), + () -> assertEquals("x", trie.get("az")), + () -> assertEquals("y", trie.get("bz")), + () -> assertNull(trie.get("c"))); + } + + /** + * Verifies that binary persistence preserves accepting leaf semantics. + */ + @Test + @Tag("persistence") + @DisplayName("Binary round trip preserves uniform subtree accepting leaf") + void shouldPreserveUniformSubtreeContractionAcrossBinaryRoundTrip() throws IOException { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder<>(String[]::new, + uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD); + builder.put("aa", "x"); + builder.put("ab", "x"); + builder.put("ba", "y"); + final FrequencyTrie original = builder.build(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + original.writeTo(outputStream, STRING_CODEC); + + final FrequencyTrie restored = FrequencyTrie + .readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC); + + assertAll("restored contraction", + () -> assertTrue(restored.root().findChild('a').acceptsRemainingInput()), + () -> assertEquals("x", restored.get("az")), + () -> assertTrue(restored.metadata().reductionSettings().contractUniformSubtrees())); + } + + /** + * Verifies that value mapping keeps accepting leaf semantics. + */ + @Test + @Tag("reduction") + @DisplayName("Value mapping preserves uniform subtree accepting leaf") + void shouldPreserveUniformSubtreeContractionWhenMappingValues() { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder<>(String[]::new, + uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD); + builder.put("aa", "x"); + builder.put("ab", "x"); + builder.put("ba", "y"); + final FrequencyTrie source = builder.build(); + + final FrequencyTrie mapped = FrequencyTrieBuilders.mapValues(source, Integer[]::new, + source.metadata().reductionSettings(), String::length); + + assertAll("mapped contraction", + () -> assertTrue(mapped.root().findChild('a').acceptsRemainingInput()), + () -> assertEquals(1, mapped.get("az"))); + } + /** * Verifies that cyclic serialized node references are rejected as invalid * serialization. diff --git a/src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java b/src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java index c7f6179..c24773c 100644 --- a/src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java +++ b/src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java @@ -69,6 +69,7 @@ import org.junit.jupiter.api.io.TempDir; @Tag("trie") @Tag("stemmer") @Tag("determinism") +@SuppressWarnings("deprecation") class FuzzStemmerAndTrieCompilationTest { /** diff --git a/src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java b/src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java index 1c409b6..085b46a 100644 --- a/src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java +++ b/src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java @@ -50,6 +50,7 @@ import org.junit.jupiter.api.Tag; @Tag("property") @Tag("patch") @Tag("stemmer") +@SuppressWarnings("deprecation") class PatchCommandEncoderProperties extends PropertyBasedTestSupport { /** diff --git a/src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java b/src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java index 88cace9..3477435 100644 --- a/src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java +++ b/src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java @@ -72,6 +72,7 @@ import org.junit.jupiter.params.provider.MethodSource; @Tag("encoding") @Tag("apply") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SuppressWarnings("deprecation") class PatchCommandEncoderTest { /** diff --git a/src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java b/src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java index 6e73a0a..7a5f412 100644 --- a/src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java +++ b/src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java @@ -56,6 +56,7 @@ import java.util.logging.Logger; *
  • {@code --reduction-mode }
    • * */ +@SuppressWarnings("deprecation") public final class RegressionArtifactGenerator { /** diff --git a/src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java b/src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java index 4b46892..3ac581e 100644 --- a/src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java +++ b/src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java @@ -49,6 +49,7 @@ import java.util.Objects; * calculation, and failure-message formatting so that regression tests stay * focused on contract verification. */ +@SuppressWarnings("deprecation") final class RegressionArtifactSupport { /** diff --git a/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java b/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java index 22e0fa8..9e8b293 100644 --- a/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java +++ b/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java @@ -92,6 +92,7 @@ import org.junit.jupiter.params.provider.MethodSource; @Tag("trie") @Tag("persistence") @TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SuppressWarnings("deprecation") final class StemmerPatchTrieLoaderTest { /** @@ -251,6 +252,72 @@ final class StemmerPatchTrieLoaderTest { StemmerPatchTrieLoader.FILENAME_REQUIRED), Arguments.of("27-load-binary-metadata-stream-null", (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null), + "inputStream"), + Arguments.of("28-load-compiled-language-settings-null-language", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled( + (StemmerPatchTrieLoader.Language) null, true, settings), + "language"), + Arguments.of("29-load-compiled-language-settings-null-settings", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, (ReductionSettings) null), + "reductionSettings"), + Arguments.of("30-load-compiled-language-mode-null-language", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled( + (StemmerPatchTrieLoader.Language) null, true, DEFAULT_REDUCTION_MODE), + "language"), + Arguments.of("31-load-compiled-language-mode-null-mode", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, (ReductionMode) null), + "reductionMode"), + Arguments.of("32-load-compiled-language-metadata-null-metadata", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, (TrieMetadata) null), + "metadata"), + Arguments.of("33-load-compiled-path-settings-null-path", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled((Path) null, true, settings), + "path"), + Arguments.of("34-load-compiled-path-settings-null-settings", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true, + (ReductionSettings) null), + "reductionSettings"), + Arguments.of("35-load-compiled-path-mode-null-mode", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true, + (ReductionMode) null), + "reductionMode"), + Arguments.of("36-load-compiled-path-metadata-null-metadata", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true, + (TrieMetadata) null), + "metadata"), + Arguments.of("37-load-compiled-string-settings-null-file", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled((String) null, true, settings), + StemmerPatchTrieLoader.FILENAME_REQUIRED), + Arguments.of("38-load-compiled-string-settings-null-settings", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true, + (ReductionSettings) null), + "reductionSettings"), + Arguments.of("39-load-compiled-string-mode-null-mode", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true, + (ReductionMode) null), + "reductionMode"), + Arguments.of("40-load-compiled-string-metadata-null-metadata", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true, + (TrieMetadata) null), + "metadata"), + Arguments.of("41-load-binary-compiled-path-null", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((Path) null), "path"), + Arguments.of("42-load-binary-compiled-path-override-null", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((Path) null, + FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX), + "path"), + Arguments.of("43-load-binary-compiled-string-null", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((String) null), + StemmerPatchTrieLoader.FILENAME_REQUIRED), + Arguments.of("44-load-binary-compiled-string-override-null", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((String) null, + FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX), + StemmerPatchTrieLoader.FILENAME_REQUIRED), + Arguments.of("45-load-binary-compiled-stream-null", + (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((InputStream) null), "inputStream")); } @@ -431,6 +498,38 @@ final class StemmerPatchTrieLoaderTest { "run"); } + /** + * Verifies that textual compiled loading overloads produce patch-command + * tries with the same stemming semantics as the canonical textual patch trie. + * + * @throws IOException if the test file cannot be written or read + */ + @Test + @DisplayName("Textual compiled load overloads must preserve stemming semantics") + void shouldLoadCompiledTrieFromTextualOverloads() throws IOException { + final Path dictionaryFile = writeDictionary(""" + run running runs runner + play playing played plays + city cities + """); + + final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE); + final FrequencyTrie expected = StemmerPatchTrieLoader.load(dictionaryFile, true, settings); + final FrequencyTrie fromPathWithSettings = StemmerPatchTrieLoader.loadCompiled( + dictionaryFile, true, settings); + final FrequencyTrie fromPathWithMode = StemmerPatchTrieLoader.loadCompiled( + dictionaryFile, true, DEFAULT_REDUCTION_MODE); + final FrequencyTrie fromStringWithSettings = StemmerPatchTrieLoader.loadCompiled( + dictionaryFile.toString(), true, settings); + final FrequencyTrie fromStringWithMode = StemmerPatchTrieLoader.loadCompiled( + dictionaryFile.toString(), true, DEFAULT_REDUCTION_MODE); + + assertCompiledTrieSemanticsEqual(expected, fromPathWithSettings, "running", "played", "cities", "run"); + assertCompiledTrieSemanticsEqual(expected, fromPathWithMode, "running", "played", "cities", "run"); + assertCompiledTrieSemanticsEqual(expected, fromStringWithSettings, "running", "played", "cities", "run"); + assertCompiledTrieSemanticsEqual(expected, fromStringWithMode, "running", "played", "cities", "run"); + } + /** * Verifies that metadata-driven loading keeps all configuration dimensions in * one explicit object and applies them during compilation. @@ -577,14 +676,28 @@ final class StemmerPatchTrieLoaderTest { StemmerPatchTrieLoader.saveBinary(original, binaryFile); final FrequencyTrie fromPath = StemmerPatchTrieLoader.loadBinary(binaryFile); final FrequencyTrie fromString = StemmerPatchTrieLoader.loadBinary(binaryFile.toString()); + final FrequencyTrie compiledFromPath = StemmerPatchTrieLoader.loadBinaryCompiled( + binaryFile); + final FrequencyTrie compiledFromString = StemmerPatchTrieLoader.loadBinaryCompiled( + binaryFile.toString()); final byte[] binaryBytes = Files.readAllBytes(binaryFile); try (InputStream inputStream = new ByteArrayInputStream(binaryBytes)) { final FrequencyTrie fromStream = StemmerPatchTrieLoader.loadBinary(inputStream); + final FrequencyTrie compiledFromStream; + try (InputStream compiledInputStream = new ByteArrayInputStream(binaryBytes)) { + compiledFromStream = StemmerPatchTrieLoader.loadBinaryCompiled(compiledInputStream); + } assertTriePatchSemanticsEqual(original, fromPath, "run", "running", "runner", "cities", "studying"); assertTriePatchSemanticsEqual(original, fromString, "run", "running", "runner", "cities", "studying"); assertTriePatchSemanticsEqual(original, fromStream, "run", "running", "runner", "cities", "studying"); + assertCompiledTrieSemanticsEqual(original, compiledFromPath, "run", "running", "runner", "cities", + "studying"); + assertCompiledTrieSemanticsEqual(original, compiledFromString, "run", "running", "runner", "cities", + "studying"); + assertCompiledTrieSemanticsEqual(original, compiledFromStream, "run", "running", "runner", "cities", + "studying"); } final TrieMetadata metadataFromPath = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile); @@ -856,6 +969,30 @@ final class StemmerPatchTrieLoaderTest { return stems; } + /** + * Reconstructs all stem candidates for the supplied word from compiled patch + * commands returned by {@link FrequencyTrie#getAll(String)}. + * + * @param trie compiled patch-command trie + * @param word surface word + * @return reconstructed stem candidates + */ + private static Set reconstructAllCompiledStemCandidates(final FrequencyTrie trie, + final String word) { + final CompiledPatchCommand[] patchCommands = trie.getAll(word); + final Set stems = new LinkedHashSet(); + + if (patchCommands == null) { + return stems; + } + + for (CompiledPatchCommand patchCommand : patchCommands) { + stems.add(patchCommand.apply(word)); + } + + return stems; + } + /** * Verifies semantic equality of two tries for the supplied words by comparing * both their raw patch arrays and reconstructed stem sets. @@ -876,6 +1013,26 @@ final class StemmerPatchTrieLoaderTest { } } + /** + * Verifies semantic equality of a textual patch trie and a compiled patch trie + * for the supplied words. + * + * @param expected reference trie with textual patch commands + * @param actual compared trie with compiled patch commands + * @param words words to verify + */ + private static void assertCompiledTrieSemanticsEqual(final FrequencyTrie expected, + final FrequencyTrie actual, final String... words) { + assertAll(() -> assertEquals(expected.metadata(), actual.metadata(), "Trie metadata must be preserved."), + () -> assertEquals(expected.traversalDirection(), actual.traversalDirection(), + "Trie traversal direction must be preserved.")); + + for (String word : words) { + assertEquals(reconstructAllStemCandidates(expected, word), reconstructAllCompiledStemCandidates(actual, word), + "Compiled patch stems must match textual patch stems for word '" + word + "'."); + } + } + /** * Opens one bundled dictionary resource. * diff --git a/src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java b/src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java index 27474cc..1b26525 100644 --- a/src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java +++ b/src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java @@ -58,6 +58,7 @@ import org.junit.jupiter.api.Tag; @Label("Stemmer patch trie properties") @Tag("property") @Tag("stemmer") +@SuppressWarnings("deprecation") class StemmerPatchTrieProperties extends PropertyBasedTestSupport { /** diff --git a/src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java b/src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java new file mode 100644 index 0000000..9986c8c --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java @@ -0,0 +1,187 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for benchmark token sequence reuse and offset behavior. + */ +@Tag("benchmark") +@Tag("unit") +@DisplayName("BenchmarkTokenSequence") +final class BenchmarkTokenSequenceTest { + + /** + * Shared token corpus used in the benchmark state. + */ + private static final String[] TOKENS = { "running", "caresses", "running", "happiness", "", "s" }; + + /** + * Fully qualified benchmark helper class name. + */ + private static final String SEQUENCE_CLASS = "org.egothor.stemmer.benchmark.BenchmarkTokenSequence"; + + /** + * Verifies first run and rewind behavior keeps token order stable. + */ + @Test + @DisplayName("should emit tokens in stable order and correct offsets") + void shouldEmitTokensInStableOrder() throws Exception { + final Object sequence = createSequence(TOKENS); + final String[] expected = TOKENS; + final Method advance = method("advance"); + final Method hasNext = method("hasNext"); + final Method currentToken = method("currentToken"); + final Method currentStartOffset = method("currentStartOffset"); + final Method currentEndOffset = method("currentEndOffset"); + + for (int index = 0; index < expected.length; index++) { + assertTrue((Boolean) advance.invoke(sequence)); + assertEquals(expected[index], new String((char[]) currentToken.invoke(sequence))); + if (index == 0) { + assertEquals(0, ((Number) currentStartOffset.invoke(sequence)).intValue()); + assertEquals("running".length(), ((Number) currentEndOffset.invoke(sequence)).intValue()); + } + } + assertFalse((Boolean) hasNext.invoke(sequence)); + assertFalse((Boolean) advance.invoke(sequence)); + } + + /** + * Verifies reset brings the sequence back to the first token. + */ + @Test + @DisplayName("should support reset and replay without allocations") + void shouldSupportResetAndReplay() throws Exception { + final Object sequence = createSequence(TOKENS); + final Method advance = method("advance"); + final Method reset = method("reset"); + + int firstPass = 0; + while ((Boolean) advance.invoke(sequence)) { + firstPass++; + } + + reset.invoke(sequence); + + int secondPass = 0; + while ((Boolean) advance.invoke(sequence)) { + secondPass++; + } + + assertEquals(firstPass, secondPass); + } + + /** + * Verifies sequence tokens can be replaced for multi-benchmark reuse. + */ + @Test + @DisplayName("should reset offsets after token set replacement") + void shouldResetOffsetsAfterTokenReplacement() throws Exception { + final Object sequence = createSequence(new String[] { "a", "bc", "def" }); + final Method setTokens = method("setTokens", String[].class); + final Method advance = method("advance"); + final Method currentToken = method("currentToken"); + + advance.invoke(sequence); + advance.invoke(sequence); + + final String[] replacement = { "xy", "z" }; + setTokens.invoke(sequence, (Object) replacement); + advance.invoke(sequence); + assertEquals("xy", new String((char[]) currentToken.invoke(sequence))); + advance.invoke(sequence); + assertEquals("z", new String((char[]) currentToken.invoke(sequence))); + } + + /** + * Verifies no mutation of original token instances in source corpus. + */ + @Test + @DisplayName("should not mutate source token references or values") + void shouldNotMutateSourceTokenValues() throws Exception { + final String[] source = { "first", "second", "third" }; + final Object sequence = createSequence(source); + final Method advance = method("advance"); + final Method currentToken = method("currentToken"); + + while ((Boolean) advance.invoke(sequence)) { + assertFalse(new String((char[]) currentToken.invoke(sequence)).isEmpty()); + } + + assertEquals("first", source[0]); + assertEquals("second", source[1]); + assertEquals("third", source[2]); + } + + /** + * Instantiates the benchmark token sequence class. + * + * @param tokens source tokens + * @return sequence instance + * @throws Exception on reflection failure + */ + private Object createSequence(final String[] tokens) throws Exception { + try { + final Class type = Class.forName(SEQUENCE_CLASS); + final Constructor constructor = type.getDeclaredConstructor(String[].class); + return constructor.newInstance((Object) tokens); + } catch (ClassNotFoundException exception) { + Assumptions.assumeTrue(false, "Benchmark token sequence class is available only when JMH sources are compiled."); + throw exception; + } + } + + /** + * Resolves a method for the token sequence class. + * + * @param name method name + * @param arguments argument types + * @return method + * @throws Exception when method is missing + */ + private Method method(final String name, final Class... arguments) throws Exception { + final Method method = Class.forName(SEQUENCE_CLASS).getDeclaredMethod(name, arguments); + method.setAccessible(true); + return method; + } +} diff --git a/src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java b/src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java new file mode 100644 index 0000000..e9a5c5a --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java @@ -0,0 +1,230 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Reflection-based tests for the Lucene-dependent benchmark token stream. + */ +@Tag("benchmark") +@Tag("unit") +@DisplayName("EnglishStemmerComparisonTokenStream") +final class EnglishStemmerComparisonTokenStreamTest { + + /** + * Fully qualified token stream class name. + */ + private static final String TOKEN_STREAM_CLASS = "org.egothor.stemmer.benchmark.EnglishStemmerComparisonTokenStream"; + + /** + * Verifies stream reuse and reset behavior through repeated iteration. + * + * @throws Exception when reflection calls fail + */ + @Test + @DisplayName("should reuse token stream without losing order or count") + void shouldReuseTokenStreamWithoutLosingOrderOrCount() throws Exception { + final Object stream = createStream(new String[] { "running", "caresses", "happiness" }); + final Class type = streamType(); + final Method increment = method(type, "incrementToken"); + final Method reset = method(type, "reset"); + final Method isDrained = method(type, "isDrained"); + final Method setTokens = method(type, "setTokens", String[].class); + + int count = consume(increment, stream); + assertEquals(3, count); + assertEquals(Boolean.TRUE, isDrained.invoke(stream)); + + reset.invoke(stream); + assertEquals(3, consume(increment, stream)); + + setTokens.invoke(stream, (Object) new String[] { "single" }); + reset.invoke(stream); + assertEquals(1, consume(increment, stream)); + } + + /** + * Verifies empty stream handling, end-of-stream, and reset behavior. + * + * @throws Exception when reflection calls fail + */ + @Test + @DisplayName("should handle empty corpus with immediate drain") + void shouldHandleEmptyCorpusWithImmediateDrain() throws Exception { + final Object stream = createStream(new String[0]); + final Class type = streamType(); + final Method increment = method(type, "incrementToken"); + final Method isDrained = method(type, "isDrained"); + final Method reset = method(type, "reset"); + + assertFalse((Boolean) increment.invoke(stream)); + assertEquals(Boolean.TRUE, isDrained.invoke(stream)); + + reset.invoke(stream); + assertFalse((Boolean) increment.invoke(stream)); + } + + /** + * Verifies the last emitted token is stable across repeated passes. + * + * @throws Exception when reflection calls fail + */ + @Test + @DisplayName("should expose stable terminal token text") + void shouldExposeStableTerminalTokenText() throws Exception { + final Object stream = createStream(new String[] { "caresses", "running", "connected" }); + final Class type = streamType(); + final Class charTermClass = Class.forName("org.apache.lucene.analysis.tokenattributes.CharTermAttribute"); + final Method increment = method(type, "incrementToken"); + final Method getAttribute = method(type, "getAttribute", false, Class.class); + final Method end = method(type, "end"); + final Method close = method(type, "close"); + final Object termAttribute = getAttribute.invoke(stream, charTermClass); + + String lastToken = null; + while ((Boolean) increment.invoke(stream)) { + lastToken = termAttribute.toString(); + } + end.invoke(stream); + close.invoke(stream); + assertEquals("connected", lastToken); + } + + /** + * Resolves and instantiate the benchmark token stream class when available. + * + * @param tokens input tokens + * @return created stream + * @throws Exception when class or constructor fails + */ + private Object createStream(final String[] tokens) throws Exception { + final Class type = streamType(); + final Constructor constructor = type.getDeclaredConstructor(String[].class); + constructor.setAccessible(true); + return constructor.newInstance((Object) tokens); + } + + /** + * Resolves the benchmark token stream class. + * + * @return stream class + * @throws Exception when class loading fails + */ + private Class streamType() throws Exception { + try { + return Class.forName(TOKEN_STREAM_CLASS); + } catch (ClassNotFoundException exception) { + Assumptions.assumeTrue(false, "Token stream class is available only when JMH sources are compiled."); + throw exception; + } + } + + /** + * Resolves a method for invocation. + * + * @param type target class + * @param name method name + * @param arguments argument types + * @return reflected method + * @throws NoSuchMethodException when method is missing + */ + private Method method(final Class type, final String name, final Class... arguments) throws NoSuchMethodException { + return method(type, name, true, arguments); + } + + /** + * Resolves a method for invocation. + * + * @param type target class + * @param name method name + * @param declared whether to require declaration in the target class + * @param arguments argument types + * @return reflected method + * @throws NoSuchMethodException when method is missing + */ + private Method method(final Class type, final String name, final boolean declared, final Class... arguments) + throws NoSuchMethodException { + final Method method = declared ? type.getDeclaredMethod(name, arguments) : type.getMethod(name, arguments); + method.setAccessible(true); + return method; + } + + /** + * Consumes stream and counts tokens. + * + * @param increment increment method + * @param stream stream object + * @return tokens emitted + * @throws Exception on reflection failure + */ + private int consume(final Method increment, final Object stream) throws Exception { + int count = 0; + while ((Boolean) increment.invoke(stream)) { + count++; + } + return count; + } + + /** + * Confirms invocation paths fail fast for invalid signatures. + */ + @Test + @DisplayName("should enforce method contract on reflection") + void shouldEnforceMethodContractOnReflection() { + Assumptions.assumeTrue(streamTypeAvailable(), "Token stream class is available only when JMH sources are compiled."); + assertThrows(NoSuchMethodException.class, () -> streamType().getDeclaredMethod("nonExistentMethod")); + } + + /** + * Checks whether the benchmark token stream class can be loaded. + * + * @return true when class is available + */ + private boolean streamTypeAvailable() { + try { + Class.forName(TOKEN_STREAM_CLASS); + return true; + } catch (ClassNotFoundException exception) { + return false; + } + } +} diff --git a/src/test/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpusTest.java b/src/test/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpusTest.java new file mode 100644 index 0000000..022e87d --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpusTest.java @@ -0,0 +1,228 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; + +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Tests dictionary-derived benchmark corpus construction. + */ +@Tag("benchmark") +@Tag("unit") +@DisplayName("LanguageBenchmarkCorpus") +final class LanguageBenchmarkCorpusTest { + + /** + * Fully qualified corpus helper class name. + */ + private static final String CORPUS_CLASS = "org.egothor.stemmer.benchmark.LanguageBenchmarkCorpus"; + + /** + * Verifies large resources use the full dictionary-derived token sequence. + * + * @throws Exception if reflection or resource loading fails + */ + @Test + @DisplayName("should use full dictionary corpus when resource is larger than the timing minimum") + void shouldUseFullDictionaryCorpusWhenResourceIsLargerThanTimingMinimum() throws Exception { + final Object fullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object timingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK); + + assertTrue(tokens(fullCorpus).length > minimumTimingTokenCount()); + assertEquals(tokens(fullCorpus).length, tokens(timingCorpus).length); + assertArrayEquals(tokens(fullCorpus), tokens(timingCorpus)); + assertArrayEquals(expectedRoots(fullCorpus), expectedRoots(timingCorpus)); + } + + /** + * Verifies small resources repeat deterministically to the minimum timing size. + * + * @throws Exception if reflection or resource loading fails + */ + @Test + @DisplayName("should repeat small dictionary corpus to timing minimum") + void shouldRepeatSmallDictionaryCorpusToTimingMinimum() throws Exception { + final Object fullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.FA_IR); + final Object timingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.FA_IR); + + assertTrue(tokens(fullCorpus).length < minimumTimingTokenCount()); + assertEquals(minimumTimingTokenCount(), tokens(timingCorpus).length); + assertEquals(tokens(fullCorpus)[0], tokens(timingCorpus)[0]); + assertEquals(expectedRoots(fullCorpus)[0], expectedRoots(timingCorpus)[0]); + assertEquals(tokens(fullCorpus)[0], tokens(timingCorpus)[tokens(fullCorpus).length]); + assertEquals(expectedRoots(fullCorpus)[0], expectedRoots(timingCorpus)[tokens(fullCorpus).length]); + } + + /** + * Verifies corpus token and expected-root arrays stay aligned. + * + * @throws Exception if reflection or resource loading fails + */ + @Test + @DisplayName("should keep token and expected-root arrays aligned") + void shouldKeepTokenAndExpectedRootArraysAligned() throws Exception { + final Object corpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.PL_PL); + + assertEquals(tokens(corpus).length, expectedRoots(corpus).length); + assertTrue(tokens(corpus).length > minimumTimingTokenCount()); + assertTrue(expectedRoots(corpus)[0].length() > 0); + } + + /** + * Verifies changed-token timing corpora exclude entries already equal to the + * expected root. + * + * @throws Exception if reflection or resource loading fails + */ + @Test + @DisplayName("should create changed-token timing corpus") + void shouldCreateChangedTokenTimingCorpus() throws Exception { + final Object corpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK); + final String[] corpusTokens = tokens(corpus); + final String[] corpusExpectedRoots = expectedRoots(corpus); + + assertTrue(corpusTokens.length > minimumTimingTokenCount()); + for (int index = 0; index < corpusTokens.length; index++) { + assertTrue(!corpusTokens[index].equals(corpusExpectedRoots[index]), + "Changed-token corpus must contain only token/root pairs where token differs from root."); + } + } + + /** + * Verifies benchmark corpora are generated once per language and reused from + * memory. + * + * @throws Exception if reflection or resource loading fails + */ + @Test + @DisplayName("should reuse cached corpus instances") + void shouldReuseCachedCorpusInstances() throws Exception { + final Object firstTimingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object secondTimingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object firstFullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object secondFullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object firstChangedCorpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK); + final Object secondChangedCorpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK); + + assertSame(firstTimingCorpus, secondTimingCorpus); + assertSame(firstFullCorpus, secondFullCorpus); + assertSame(firstChangedCorpus, secondChangedCorpus); + } + + /** + * Invokes a static corpus factory. + * + * @param methodName factory method name + * @param language Radixor language + * @return corpus record instance + * @throws Exception if reflection fails + */ + private Object invokeCorpus(final String methodName, final StemmerPatchTrieLoader.Language language) + throws Exception { + final Class type = corpusType(); + final Method method = type.getDeclaredMethod(methodName, StemmerPatchTrieLoader.Language.class); + method.setAccessible(true); + return method.invoke(null, language); + } + + /** + * Reads corpus tokens. + * + * @param corpus corpus record instance + * @return token array + * @throws Exception if reflection fails + */ + private String[] tokens(final Object corpus) throws Exception { + return stringArray(corpus, "tokens"); + } + + /** + * Reads corpus expected roots. + * + * @param corpus corpus record instance + * @return expected-root array + * @throws Exception if reflection fails + */ + private String[] expectedRoots(final Object corpus) throws Exception { + return stringArray(corpus, "expectedRoots"); + } + + /** + * Reads a string-array record component. + * + * @param corpus corpus record instance + * @param methodName accessor name + * @return string array + * @throws Exception if reflection fails + */ + private String[] stringArray(final Object corpus, final String methodName) throws Exception { + final Method method = corpus.getClass().getDeclaredMethod(methodName); + method.setAccessible(true); + return (String[]) method.invoke(corpus); + } + + /** + * Reads the minimum timing token count constant. + * + * @return minimum timing token count + * @throws Exception if reflection fails + */ + private int minimumTimingTokenCount() throws Exception { + final java.lang.reflect.Field field = corpusType().getDeclaredField("MINIMUM_TIMING_TOKEN_COUNT"); + field.setAccessible(true); + return ((Number) field.get(null)).intValue(); + } + + /** + * Resolves the benchmark corpus helper class. + * + * @return corpus helper type + * @throws Exception if class loading fails + */ + private Class corpusType() throws Exception { + try { + return Class.forName(CORPUS_CLASS); + } catch (ClassNotFoundException exception) { + Assumptions.assumeTrue(false, "Language benchmark corpus is available only when JMH sources are compiled."); + throw exception; + } + } +} diff --git a/src/test/java/org/egothor/stemmer/benchmark/LucenePorterStemmerCopiedTest.java b/src/test/java/org/egothor/stemmer/benchmark/LucenePorterStemmerCopiedTest.java new file mode 100644 index 0000000..36c3daf --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/LucenePorterStemmerCopiedTest.java @@ -0,0 +1,127 @@ +/****************************************************************************** + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the generated Lucene Porter stemmer benchmark adapter. + */ +@Tag("benchmark") +@Tag("unit") +@DisplayName("LucenePorterStemmerCopied") +final class LucenePorterStemmerCopiedTest { + + /** + * Fully qualified benchmark class under test. + */ + private static final String STEMMER_CLASS = "org.egothor.stemmer.benchmark.LucenePorterStemmerCopied"; + + /** + * Verifies repeated invocations for representative tokens are deterministic. + */ + @Test + @DisplayName("should produce stable stems for representative tokens") + void shouldProduceStableStemsForRepresentativeTokens() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + final String[] tokens = { "running", "caresses", "happiness", "connected", "dancing", "" }; + for (String token : tokens) { + final String first = (String) stemMethod.invoke(stemmer, token); + final String second = (String) stemMethod.invoke(stemmer, token); + assertEquals(first, second); + assertNotNull(first); + } + } + + /** + * Verifies short tokens are accepted and remain non-null. + */ + @Test + @DisplayName("should return non-null stems for empty and short tokens") + void shouldReturnNonNullStemsForEmptyAndShortTokens() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + assertEquals("", stemMethod.invoke(stemmer, "")); + assertEquals("a", stemMethod.invoke(stemmer, "a")); + assertEquals("go", stemMethod.invoke(stemmer, "go")); + } + + /** + * Verifies mutable reuse on a single benchmark instance. + */ + @Test + @DisplayName("should preserve state across many repeated calls on one instance") + void shouldPreserveStateAcrossManyRepeatedCallsOnOneInstance() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + final String first = (String) stemMethod.invoke(stemmer, "connected"); + for (int index = 0; index < 64; index++) { + assertEquals(first, stemMethod.invoke(stemmer, "connected")); + } + } + + /** + * Instantiates the benchmark copied Lucene Porter class if it is present. + * + * @return benchmark stemmer instance + * @throws Exception when reflective creation fails + */ + private Object createStemmer() throws Exception { + try { + return Class.forName(STEMMER_CLASS).getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException exception) { + Assumptions.assumeTrue(false, "Benchmark Lucene Porter class is available only when JMH sources are compiled."); + throw exception; + } + } + + /** + * Resolves the benchmark stem method. + * + * @return `stem` reflection handle + * @throws Exception when reflective lookup fails + */ + private Method stemMethod() throws Exception { + return Class.forName(STEMMER_CLASS).getDeclaredMethod("stem", String.class); + } +} diff --git a/src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java b/src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java new file mode 100644 index 0000000..1c5fbc4 --- /dev/null +++ b/src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java @@ -0,0 +1,148 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.lang.reflect.Method; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the benchmark Paice/Husk Lancaster implementation. + */ +@Tag("benchmark") +@Tag("unit") +@DisplayName("PaiceHuskLancasterStemmer") +final class PaiceHuskLancasterStemmerTest { + + /** + * Benchmark class under test. + */ + private static final String STEMMER_CLASS = "org.egothor.stemmer.benchmark.PaiceHuskLancasterStemmer"; + + /** + * Expected outputs used to verify deterministic stems for this benchmark adapter. + */ + private static final String[][] SAMPLE_STEMS = { + { "running", "run" }, + { "caresses", "cares" }, + { "happiness", "happi" }, + { "connected", "connect" }, + { "dancing", "danc" }, + { "happy", "happy" } + }; + + /** + * Verifies selected representative words produce stable stems. + */ + @Test + @DisplayName("should produce stable benchmark stems for representative words") + void shouldProduceStableStemsForRepresentativeWords() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + for (String[] sample : SAMPLE_STEMS) { + assertEquals(sample[1], stemMethod.invoke(stemmer, sample[0])); + } + } + + /** + * Verifies short and empty inputs remain stable and non-null. + */ + @Test + @DisplayName("should handle empty and short tokens without null output") + void shouldHandleEmptyAndShortTokensWithoutNullOutput() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + assertEquals("", stemMethod.invoke(stemmer, "")); + assertEquals("a", stemMethod.invoke(stemmer, "a")); + assertEquals("x", stemMethod.invoke(stemmer, "x")); + } + + /** + * Verifies that one mutable instance can be reused. + */ + @Test + @DisplayName("should be reusable across many repeated calls") + void shouldBeReusableAcrossRepeatedCalls() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + final String first = (String) stemMethod.invoke(stemmer, "running"); + final String second = (String) stemMethod.invoke(stemmer, "running"); + assertEquals(first, second); + } + + /** + * Verifies null input handling for normal integration path expectations. + */ + @Test + @DisplayName("should return null only for null inputs") + void shouldReturnNullOnlyForNullInputs() throws Exception { + final Object stemmer = createStemmer(); + final Method stemMethod = stemMethod(); + + assertEquals("running", stemMethod.invoke(stemmer, "running")); + assertEquals(null, stemMethod.invoke(stemmer, new Object[] { null })); + assertNotNull(stemMethod.invoke(stemmer, "connected")); + } + + /** + * Creates a benchmark stemmer instance. + * + * @return benchmark stemmer instance + * @throws Exception when reflection fails + */ + private Object createStemmer() throws Exception { + try { + return Class.forName(STEMMER_CLASS).getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException exception) { + Assumptions.assumeTrue(false, "Benchmark Paice/Husk stemmer is available only when JMH sources are compiled."); + throw exception; + } + } + + /** + * Resolves the stem method for benchmark execution. + * + * @return stem method + * @throws Exception when lookup fails + */ + private Method stemMethod() throws Exception { + return Class.forName(STEMMER_CLASS).getDeclaredMethod("stem", String.class); + } +} diff --git a/tools/generate-pages-badges.py b/tools/generate-pages-badges.py index 0e53fd3..c72e50c 100755 --- a/tools/generate-pages-badges.py +++ b/tools/generate-pages-badges.py @@ -6,7 +6,6 @@ This script derives compact machine-readable badge payloads from: - JaCoCo XML coverage report - PIT mutation testing XML report -- JMH CSV benchmark report The generated JSON files are intended to be consumed by Shields endpoint badges. """ @@ -14,7 +13,6 @@ The generated JSON files are intended to be consumed by Shields endpoint badges. from __future__ import annotations import argparse -import csv import json import os from pathlib import Path @@ -38,8 +36,8 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--jmh-csv", - required=True, - help="Path to the JMH CSV report." + required=False, + help="Deprecated compatibility option. JMH speed badges are no longer generated." ) parser.add_argument( "--run-metrics-dir", @@ -60,6 +58,12 @@ def write_json(target: Path, payload: dict[str, object]) -> None: target.write_text(json.dumps(payload, indent=2) + os.linesep, encoding="utf-8") +def remove_file_if_present(target: Path) -> None: + """Remove a previously generated file when it is present.""" + if target.is_file(): + target.unlink() + + def unavailable_payload(label: str) -> dict[str, object]: """Create a standard payload for unavailable metrics.""" return { @@ -83,19 +87,6 @@ def color_for_percentage(value: float) -> str: return "red" -def color_for_speedup(value: float) -> str: - """Select a badge color for a speedup factor.""" - if value >= 4.0: - return "brightgreen" - if value >= 3.0: - return "green" - if value >= 2.0: - return "yellow" - if value >= 1.0: - return "orange" - return "red" - - def coverage_payload(jacoco_xml: Path) -> dict[str, object]: """Build a line coverage badge payload from a JaCoCo XML report.""" if not jacoco_xml.is_file(): @@ -158,94 +149,27 @@ def mutation_payload(pit_xml: Path) -> dict[str, object]: } -def parse_family_count(row: dict[str, str]) -> int: - """Extract the JMH familyCount parameter from a CSV row.""" - for key, value in row.items(): - if key.startswith("Param: ") and key.endswith("familyCount"): - try: - return int(value) - except (TypeError, ValueError): - return -1 - return -1 - - -def benchmark_payload(jmh_csv: Path) -> dict[str, object]: - """Build a benchmark speedup badge payload from a JMH CSV report.""" - if not jmh_csv.is_file(): - return unavailable_payload("english benchmark") - - with jmh_csv.open("r", encoding="utf-8", newline="") as input_file: - rows = list(csv.DictReader(input_file)) - - if not rows: - return unavailable_payload("english benchmark") - - relevant_rows: list[tuple[int, str, float]] = [] - for row in rows: - benchmark = row.get("Benchmark", "") - if not benchmark.endswith( - "EnglishStemmerComparisonBenchmark.radixorUsUkProfiPreferredStem" - ) and not benchmark.endswith( - "EnglishStemmerComparisonBenchmark.snowballOriginalPorter" - ): - continue - - try: - score = float(row["Score"]) - except (KeyError, TypeError, ValueError): - continue - - relevant_rows.append((parse_family_count(row), benchmark, score)) - - if not relevant_rows: - return unavailable_payload("english benchmark") - - best_family_count = max(family_count for family_count, _, _ in relevant_rows) - radixor_score = None - porter_score = None - - for family_count, benchmark, score in relevant_rows: - if family_count != best_family_count: - continue - if benchmark.endswith(".radixorUsUkProfiPreferredStem"): - radixor_score = score - elif benchmark.endswith(".snowballOriginalPorter"): - porter_score = score - - if radixor_score is None or porter_score is None or porter_score <= 0.0: - return unavailable_payload("english benchmark") - - # score is time for the batch processing, i.e. longer => slower, i.e. speedup is porter/radixor - speedup = porter_score / radixor_score - family_suffix = "" if best_family_count < 0 else f" ({best_family_count})" - return { - "schemaVersion": 1, - "label": "english benchmark", - "message": f"{speedup:.1f}x vs Porter{family_suffix}", - "color": color_for_speedup(speedup) - } - - def main() -> int: """Generate all requested badge metadata files.""" arguments = parse_args() jacoco_xml = Path(arguments.jacoco_xml) pit_xml = Path(arguments.pit_xml) - jmh_csv = Path(arguments.jmh_csv) run_metrics_dir = Path(arguments.run_metrics_dir) latest_metrics_dir = Path(arguments.latest_metrics_dir) payloads = { "coverage-badge.json": coverage_payload(jacoco_xml), - "pitest-badge.json": mutation_payload(pit_xml), - "jmh-badge.json": benchmark_payload(jmh_csv) + "pitest-badge.json": mutation_payload(pit_xml) } for file_name, payload in payloads.items(): write_json(run_metrics_dir / file_name, payload) write_json(latest_metrics_dir / file_name, payload) + remove_file_if_present(run_metrics_dir / "jmh-badge.json") + remove_file_if_present(latest_metrics_dir / "jmh-badge.json") + return 0