+ * The sequence keeps stable token ordering and offset progression while avoiding + * per-token object creation during iteration. + *
+ */ +final class BenchmarkTokenSequence { + + /** + * Shared backing corpus as character arrays. + */ + private char[][] tokenCharacters; + + /** + * Number of active tokens in the sequence. + */ + private int tokenCount; + + /** + * Cursor index for the currently emitted token. + */ + private int cursor; + + /** + * Current token character array. + */ + private char[] currentToken; + + /** + * Start offset of the current token. + */ + private int currentStartOffset; + + /** + * End offset of the current token. + */ + private int currentEndOffset; + + /** + * Offset of the next token start. + */ + private int nextOffset; + + /** + * Creates a reusable token sequence. + * + * @param tokens token corpus source + */ + BenchmarkTokenSequence(final String[] tokens) { + setTokens(tokens); + } + + /** + * Sets a new token corpus for this sequence. + * + *+ * The sequence stores copied character arrays so token reads can be reused + * without creating per-token objects during benchmark iteration. + *
+ * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + Objects.requireNonNull(tokens, "tokens"); + this.tokenCharacters = new char[tokens.length][]; + for (int index = 0; index < tokens.length; index++) { + final String token = Objects.requireNonNull(tokens[index], "tokens[" + index + "]"); + this.tokenCharacters[index] = token.toCharArray(); + } + + this.tokenCount = this.tokenCharacters.length; + reset(); + } + + /** + * Resets stream position for reuse. + */ + void reset() { + this.cursor = 0; + this.nextOffset = 0; + this.currentStartOffset = 0; + this.currentEndOffset = 0; + this.currentToken = null; + } + + /** + * Returns whether at least one token remains in the sequence. + * + * @return true if a token can be emitted + */ + boolean hasNext() { + return this.cursor < this.tokenCount; + } + + /** + * Advances to the next token. + * + * @return true if a token was emitted + */ + boolean advance() { + if (!hasNext()) { + return false; + } + + final char[] token = this.tokenCharacters[this.cursor]; + this.currentToken = token; + this.currentStartOffset = this.nextOffset; + this.currentEndOffset = this.currentStartOffset + token.length; + this.nextOffset = this.currentEndOffset + 1; + this.cursor++; + return true; + } + + /** + * Returns the current token in the sequence. + * + * @return current token character array + */ + char[] currentToken() { + return this.currentToken; + } + + /** + * Returns current token start offset for token stream attributes. + * + * @return start offset + */ + int currentStartOffset() { + return this.currentStartOffset; + } + + /** + * Returns current token end offset for token stream attributes. + * + * @return end offset + */ + int currentEndOffset() { + return this.currentEndOffset; + } + + /** + * Returns final stream offset value used by {@code end()}. + * + * @return final offset + */ + int endOffset() { + return this.nextOffset > 0 ? this.nextOffset - 1 : 0; + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java new file mode 100644 index 0000000..60fb354 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java @@ -0,0 +1,146 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +/** + * Reusable Lucene {@link TokenStream} backed by a deterministic token array. + * + *+ * Instances are mutable and intended for one JMH worker thread. The stream + * copies configured token text into reusable character storage so benchmark + * iteration can replay the same token sequence without mutating the shared + * source array. + *
+ */ +final class BenchmarkTokenStream extends TokenStream { + + /** + * Current token text attribute. + */ + private final CharTermAttribute charTermAttribute; + + /** + * Offset attribute used by Lucene filters that inspect offsets. + */ + private final OffsetAttribute offsetAttribute; + + /** + * Position increment attribute for one-token-at-a-time streams. + */ + private final PositionIncrementAttribute positionIncrementAttribute; + + /** + * Reusable token sequence. + */ + private final BenchmarkTokenSequence tokenSequence; + + /** + * Creates a stream over the supplied tokens. + * + * @param tokens initial token corpus + */ + BenchmarkTokenStream(final String[] tokens) { + this.tokenSequence = new BenchmarkTokenSequence(tokens); + this.charTermAttribute = addAttribute(CharTermAttribute.class); + this.offsetAttribute = addAttribute(OffsetAttribute.class); + this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + } + + /** + * Replaces the configured token corpus. + * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + this.tokenSequence.setTokens(tokens); + } + + /** + * Returns whether all configured tokens have been emitted. + * + * @return {@code true} after the current pass is exhausted + */ + boolean isDrained() { + return !this.tokenSequence.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean incrementToken() throws IOException { + if (!this.tokenSequence.advance()) { + return false; + } + + clearAttributes(); + final char[] token = this.tokenSequence.currentToken(); + this.charTermAttribute.copyBuffer(token, 0, token.length); + this.positionIncrementAttribute.setPositionIncrement(1); + this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset()); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public void reset() throws IOException { + super.reset(); + this.tokenSequence.reset(); + } + + /** + * {@inheritDoc} + */ + @Override + public void end() throws IOException { + super.end(); + final int endOffset = this.tokenSequence.endOffset(); + this.offsetAttribute.setOffset(endOffset, endOffset); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + super.close(); + this.charTermAttribute.setEmpty(); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java index bb99a19..0c302f4 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java @@ -30,43 +30,24 @@ ******************************************************************************/ package org.egothor.stemmer.benchmark; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; +import java.io.IOException; + +import org.egothor.stemmer.StemmerPatchTrieLoader; /** * Builds a deterministic English token corpus for side-by-side stemming - * benchmarks. + * benchmarks from the bundled Radixor English dictionary resource. * *- * The generated corpus mixes: - *
- *- * The goal is not to simulate natural language frequency distribution exactly, - * but to provide a stable and reproducible comparison workload for benchmark - * runs and regression tracking. + * The dictionary resource stores the expected stem as the first tab-separated + * field on each line and its surface variants on the same line. This helper + * uses only token/root pairs where the token differs from the expected root for + * timing. Resources smaller than the shared timing minimum are repeated + * deterministically by {@link LanguageBenchmarkCorpus}. *
*/ final class EnglishComparisonCorpus { - /** - * Canonical lexical bases used to generate the token workload. - */ - private static final String[] BASES = { "analyze", "analyse", "color", "colour", "center", "centre", "organize", - "organise", "optimize", "optimise", "characterize", "characterise", "connect", "construct", "compute", - "design", "develop", "engineer", "govern", "improve", "index", "inform", "manage", "model", "observe", - "operate", "perform", "predict", "prepare", "process", "project", "protect", "publish", "query", "reduce", - "refresh", "render", "resolve", "return", "search", "select", "signal", "store", "structure", "support", - "transform", "update", "validate", "value" }; - /** * Utility class. */ @@ -77,64 +58,21 @@ final class EnglishComparisonCorpus { /** * Creates a deterministic token corpus for English stemming comparison. * - * @param familyCount number of generated lexical families * @return token array in stable order + * @throws IOException if the bundled English resource cannot be read */ - static String[] createTokens(final int familyCount) { - if (familyCount < 1) { - throw new IllegalArgumentException("familyCount must be at least 1."); - } - - final List+ * The measured stemmer always uses {@link CompiledPatchCommand} values. Quality + * is evaluated against the complete English dictionary corpus, while speed is + * measured over the complete changed-token English corpus used by the comparison + * benchmarks. + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class EnglishRadixorDictionaryCoverageBenchmark { + + /** + * Shared benchmark state for one dictionary-row coverage percentage. + */ + @State(Scope.Benchmark) + public static class CoverageState { + + /** + * Percentage of parsed English dictionary rows used to build the Radixor trie. + */ + @Param({ "100", "90", "80", "70", "60", "50", "40", "30", "20", "10" }) + public int coveragePercent; + + /** + * Full English corpus used for exact-root accounting. + */ + private LanguageBenchmarkCorpus.Corpus fullCorpus; + + /** + * Complete changed-token English corpus used for speed measurement. + */ + private LanguageBenchmarkCorpus.Corpus changedCorpus; + + /** + * Radixor stemmer backed by a trie built from selected dictionary rows. + */ + private RadixorBenchmarkStemmer stemmer; + + /** + * Parsed dictionary row count before deterministic coverage selection. + */ + private int totalRowCount; + + /** + * Selected dictionary row count for the configured coverage percentage. + */ + private int selectedRowCount; + + /** + * Builds the reduced dictionary trie and shared corpora before measurement. + * + * @throws IOException if the English dictionary resource cannot be read + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final List- * The benchmark processes the same deterministic token array with: + * The comparison uses one shared changed-token dictionary array for all methods: *
*- * This benchmark compares throughput on a shared workload. It does not imply - * that the algorithms are linguistically equivalent. - *
*/ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 1) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) public class EnglishStemmerComparisonBenchmark { /** - * Shared benchmark data. + * Shared, parameterized benchmark corpus state. */ @State(Scope.Benchmark) public static class SharedState { /** - * Number of generated lexical families. - */ - @Param({ "1000", "5000" }) - public int familyCount; - - /** - * Token workload processed by all compared stemmers. + * Shared deterministic token corpus. */ private String[] tokens; /** - * Radixor trie loaded from the bundled professional English dictionary. + * Radixor benchmark adapter for the US/UK benchmark corpus. */ - private FrequencyTrie+ * The {@code String[]} to Lucene character-buffer conversion is deliberately + * performed every time so TokenFilter benchmarks include the cost of adapting + * the benchmark's canonical string corpus to Lucene's mutable token + * attributes. + *
+ * + * @param tokens benchmark token corpus + */ + void configure(final String[] tokens) throws IOException { + this.porterStemFilterInput.setTokens(tokens); + this.kStemFilterInput.setTokens(tokens); + this.englishMinimalStemFilterInput.setTokens(tokens); + this.englishPossessiveFilterInput.setTokens(tokens); + + this.porterStemFilter.reset(); + this.kStemFilter.reset(); + this.englishMinimalStemFilter.reset(); + this.englishPossessiveFilter.reset(); + } + + /** + * Reuses one mutable filter stream and returns all emitted tokens to blackhole. + * + * @param stream benchmark token stream with configured filter + * @param term token text attribute + * @param blackhole sink + * @throws IOException on token stream failure + */ + private static void consume(final TokenStream stream, final CharTermAttribute term, final Blackhole blackhole) + throws IOException { + while (stream.incrementToken()) { + blackhole.consume(term.toString()); + } + stream.end(); + } + + /** + * Executes Porter filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runPorterStemFilter(final Blackhole blackhole) throws IOException { + consume(this.porterStemFilter, this.porterStemFilterTerm, blackhole); + } + + /** + * Executes KStem filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runKStemFilter(final Blackhole blackhole) throws IOException { + consume(this.kStemFilter, this.kStemTerm, blackhole); + } + + /** + * Executes English minimal filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runEnglishMinimalStemFilter(final Blackhole blackhole) throws IOException { + consume(this.englishMinimalStemFilter, this.englishMinimalTerm, blackhole); + } + + /** + * Executes English possessive filter over the shared corpus. + * + * @param blackhole sink + * @throws IOException if tokenization fails + */ + void runEnglishPossessiveFilter(final Blackhole blackhole) throws IOException { + consume(this.englishPossessiveFilter, this.englishPossessiveTerm, blackhole); } } /** * Measures Radixor preferred-result stemming throughput. * - * @param sharedState shared benchmark data - * @param blackhole sink preventing dead-code elimination + *+ * This path uses a single shared dictionary lookup and patch application. + *
+ * + * @param sharedState shared corpus and trie + * @param blackhole result sink */ @Benchmark public void radixorUsUkProfiPreferredStem(final SharedState sharedState, final Blackhole blackhole) { final String[] tokens = sharedState.tokens; - final FrequencyTrie- * Snowball English is the newer English stemmer commonly referred to as - * Porter2. + * This uses Snowball classic Porter as a direct stemmer API call and includes + * no Lucene token stream integration overhead. *
* - * @param sharedState shared benchmark data - * @param snowballState reusable Snowball stemmers - * @param blackhole sink preventing dead-code elimination + * @param sharedState shared corpus + * @param stemmerState reusable Snowball adapter state + * @param blackhole result sink */ @Benchmark - public void snowballEnglishPorter2(final SharedState sharedState, final SnowballState snowballState, + public void snowballOriginalPorter(final SharedState sharedState, final DirectStemmerState stemmerState, final Blackhole blackhole) { final String[] tokens = sharedState.tokens; - final SnowballStemmerAdapter stemmer = snowballState.englishStemmer; + final SnowballStemmerAdapter stemmer = stemmerState.porterStemmer; for (String token : tokens) { blackhole.consume(stemmer.stem(token)); } } + + /** + * Measures Snowball English (Porter2) direct API throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable Snowball adapter state + * @param blackhole result sink + */ + @Benchmark + public void snowballEnglishPorter2(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final SnowballStemmerAdapter stemmer = stemmerState.englishPorterStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Lucene generated Porter stemmer API throughput. + * + *+ * This path is a generated copy of Lucene's package-private PorterStemmer + * class, compiled into the JMH source set only. + *
+ * + * @param sharedState shared corpus + * @param stemmerState reusable Lucene copied API state + * @param blackhole result sink + */ + @Benchmark + public void lucenePorterStemmerCopied(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final LucenePorterStemmerCopied stemmer = stemmerState.lucenePorter; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Lucene Porter token-filter integration throughput. + * + *+ * This includes stream, reusable token attributes, and filter overhead and is + * not equivalent to a direct API stemmer call. + *
+ * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void lucenePorterStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runPorterStemFilter(blackhole); + } + + /** + * Measures Lucene KStem integration-path throughput. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneKStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runKStemFilter(blackhole); + } + + /** + * Measures Lucene EnglishMinimal integration-path throughput. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneEnglishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runEnglishMinimalStemFilter(blackhole); + } + + /** + * Measures benchmark-only Paice/Husk Lancaster throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable Paice/Husk instance + * @param blackhole sink + */ + @Benchmark + public void paiceHuskLancaster(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final PaiceHuskLancasterStemmer stemmer = stemmerState.paiceHuskLancaster; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Measures Apache OpenNLP Porter stemming throughput. + * + * @param sharedState shared corpus + * @param stemmerState reusable OpenNLP Porter instance + * @param blackhole sink + */ + @Benchmark + public void opennlpPorterStemmer(final SharedState sharedState, final DirectStemmerState stemmerState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final opennlp.tools.stemmer.PorterStemmer stemmer = stemmerState.openNlpPorterStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token).toString()); + } + } + + /** + * Measures Lucene EnglishPossessiveFilter as a narrow possessive-removal + * baseline. + * + * @param sharedState shared corpus + * @param filterState reusable filter state + * @param blackhole sink + * @throws IOException if token stream fails + */ + @Benchmark + public void luceneEnglishPossessiveFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.configure(sharedState.tokens); + filterState.runEnglishPossessiveFilter(blackhole); + } } diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java new file mode 100644 index 0000000..1fdad74 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java @@ -0,0 +1,258 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Emits exact-root agreement metrics for the canonical English badge pair. + * + *+ * This class is deliberately named so the existing focused include pattern for + * English stemmer comparison benchmarks includes it. The benchmark methods are + * separate from throughput methods so equality checks do not contaminate timing + * scores. + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class EnglishStemmerComparisonBenchmarkQuality { + + /** + * Shared English quality corpus and stemmer state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Complete English resource-derived corpus. + */ + private LanguageBenchmarkCorpus.Corpus corpus; + + /** + * Compiled Radixor English trie. + */ + private RadixorBenchmarkStemmer radixorStemmer; + + /** + * Reusable Snowball Porter adapter. + */ + private SnowballStemmerAdapter porterStemmer; + + /** + * Initializes quality resources. + * + * @throws IOException if corpus or trie loading fails + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.corpus = LanguageBenchmarkCorpus.createFullCorpus(StemmerPatchTrieLoader.Language.US_UK); + this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + StemmerPatchTrieLoader.Language.US_UK, true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + this.porterStemmer = new SnowballStemmerAdapter(porterStemmer::new); + } + } + + /** + * JMH auxiliary counters for exact-root agreement. + */ + @State(Scope.Thread) + @AuxCounters(AuxCounters.Type.EVENTS) + public static class AccuracyCounters { + + /** + * Number of exact-root matches. + */ + public long correctMatches; + + /** + * Number of evaluated tokens. + */ + public long evaluatedTokens; + + /** + * Number of exact-root matches where the input token differs from the + * expected root. + */ + public long changedCorrectMatches; + + /** + * Number of evaluated tokens where the input token differs from the expected + * root. + */ + public long changedEvaluatedTokens; + + /** + * Number of exact-root matches where the input token is already the expected + * root. + */ + public long rootPreservedMatches; + + /** + * Number of evaluated tokens where the input token is already the expected + * root. + */ + public long rootEvaluatedTokens; + + /** + * Resets counters before each measured iteration. + */ + @Setup(Level.Iteration) + public void reset() { + this.correctMatches = 0L; + this.evaluatedTokens = 0L; + this.changedCorrectMatches = 0L; + this.changedEvaluatedTokens = 0L; + this.rootPreservedMatches = 0L; + this.rootEvaluatedTokens = 0L; + } + } + + /** + * Evaluates exact-root agreement for the canonical Radixor badge method. + * + * @param sharedState shared English quality state + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + @Benchmark + public int radixorUsUkProfiPreferredStemAccuracy(final SharedState sharedState, + final AccuracyCounters counters, final Blackhole blackhole) { + return evaluate(sharedState.corpus, sharedState.radixorStemmer::stem, counters, blackhole); + } + + /** + * Evaluates exact-root agreement for the canonical Snowball Porter badge + * method. + * + * @param sharedState shared English quality state + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + @Benchmark + public int snowballOriginalPorterAccuracy(final SharedState sharedState, + final AccuracyCounters counters, final Blackhole blackhole) { + return evaluate(sharedState.corpus, sharedState.porterStemmer::stem, counters, blackhole); + } + + /** + * Evaluates one stemmer against the expected roots. + * + * @param corpus token/root corpus + * @param stemmer stemmer under evaluation + * @param counters JMH auxiliary counters + * @param blackhole result sink + * @return exact-root match count + */ + private static int evaluate(final LanguageBenchmarkCorpus.Corpus corpus, final Stemmer stemmer, + final AccuracyCounters counters, final Blackhole blackhole) { + Objects.requireNonNull(corpus, "corpus"); + Objects.requireNonNull(stemmer, "stemmer"); + + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + final String[] tokens = corpus.tokens(); + final String[] expectedRoots = corpus.expectedRoots(); + for (int index = 0; index < tokens.length; index++) { + final String token = tokens[index]; + final String expectedRoot = expectedRoots[index]; + final String actual = stemmer.stem(token); + blackhole.consume(actual); + final boolean exact = Objects.equals(expectedRoot, actual); + if (exact) { + correct++; + } + if (Objects.equals(token, expectedRoot)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + + counters.correctMatches += correct; + counters.evaluatedTokens += tokens.length; + counters.changedCorrectMatches += changedCorrect; + counters.changedEvaluatedTokens += changedEvaluated; + counters.rootPreservedMatches += rootPreserved; + counters.rootEvaluatedTokens += rootEvaluated; + return correct; + } + + /** + * Direct stemmer function. + */ + @FunctionalInterface + private interface Stemmer { + + /** + * Produces one stem. + * + * @param token input token + * @return produced stem + */ + String stem(String token); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java new file mode 100644 index 0000000..a5ced0f --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +/** + * Reusable token stream driven by a deterministic token corpus. + * + *+ * The stream emits each token from a shared array and supports repeated + * {@link #reset()} + {@link #incrementToken()} cycles without per-token + * object allocation. + *
+ */ +final class EnglishStemmerComparisonTokenStream extends TokenStream { + + /** + * Current token text. + */ + private final CharTermAttribute charTermAttribute; + + /** + * Token offsets for benchmark stream compliance. + */ + private final OffsetAttribute offsetAttribute; + + /** + * Position increment attribute for benchmark stream compliance. + */ + private final PositionIncrementAttribute positionIncrementAttribute; + + /** + * Reusable token source. + */ + private final BenchmarkTokenSequence tokenSequence; + + /** + * Creates a deterministic token stream for benchmark reuse. + * + * @param tokens tokens emitted by the stream + */ + EnglishStemmerComparisonTokenStream(final String[] tokens) { + this.tokenSequence = new BenchmarkTokenSequence(tokens); + this.charTermAttribute = addAttribute(CharTermAttribute.class); + this.offsetAttribute = addAttribute(OffsetAttribute.class); + this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + } + + /** + * Replaces the token corpus for this stream. + * + * @param tokens new token corpus + */ + void setTokens(final String[] tokens) { + this.tokenSequence.setTokens(tokens); + } + + /** + * Returns whether the stream is drained and ready to be exhausted. + * + * @return true if all configured tokens were consumed + */ + boolean isDrained() { + return !this.tokenSequence.hasNext(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean incrementToken() throws IOException { + if (!this.tokenSequence.advance()) { + return false; + } + + clearAttributes(); + final char[] token = this.tokenSequence.currentToken(); + this.charTermAttribute.copyBuffer(token, 0, token.length); + this.positionIncrementAttribute.setPositionIncrement(1); + this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset()); + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public void reset() throws IOException { + super.reset(); + this.tokenSequence.reset(); + } + + /** + * {@inheritDoc} + */ + @Override + public void end() throws IOException { + super.end(); + final int endOffset = this.tokenSequence.endOffset(); + this.offsetAttribute.setOffset(endOffset, endOffset); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + super.close(); + this.charTermAttribute.setEmpty(); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java b/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java index ee382e7..fc4416d 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java @@ -65,6 +65,7 @@ import org.openjdk.jmh.infra.Blackhole; @OutputTimeUnit(TimeUnit.NANOSECONDS) @Warmup(iterations = 3, time = 1) @Measurement(iterations = 5, time = 1) +@SuppressWarnings("deprecation") public class FrequencyTrieLookupBenchmark { /** diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java b/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java new file mode 100644 index 0000000..fc5c9a4 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java @@ -0,0 +1,445 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Locale; +import java.util.Objects; +import java.util.zip.GZIPInputStream; + +import org.egothor.stemmer.StemmerPatchTrieLoader; + +/** + * Builds deterministic language-specific benchmark corpora from bundled + * Radixor dictionary resources. + * + *+ * Corpus construction is setup work only. It is intentionally based on the same + * resource that backs the Radixor benchmark path so every competitor for a + * language consumes the same changed-token timing workload, while quality + * benchmarks can still use the complete dictionary workload. + *
+ */ +final class LanguageBenchmarkCorpus { + + /** + * Minimum token count for timing benchmark operations. + */ + static final int MINIMUM_TIMING_TOKEN_COUNT = 5_000; + + /** + * Shared timing corpora keyed by bundled Radixor language. + */ + private static final Map+ * Only token/root pairs where the token differs from the expected root are + * included. Smaller changed-token resources are repeated in stable order until + * the timing corpus reaches 5,000 tokens. + *
+ * + * @param language bundled Radixor language + * @return token array containing changed-token dictionary entries, repeated + * only when the changed-token resource is smaller than 5,000 tokens + * @throws IOException if the resource cannot be read + */ + static String[] createTokens(final StemmerPatchTrieLoader.Language language) throws IOException { + return createChangedCorpus(language).tokens(); + } + + /** + * Creates a deterministic changed-token timing corpus from a bundled language + * dictionary. + * + * @param language bundled Radixor language + * @return changed-token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createChangedCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedChangedCorpus(language); + } + + /** + * Creates a deterministic full-dictionary timing corpus and expected root + * array from a bundled language dictionary. + * + * @param language bundled Radixor language + * @return token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedCorpus(TIMING_CORPORA, language, true); + } + + /** + * Creates a deterministic full-dictionary timing corpus and expected root + * array from a bundled language dictionary. + * + *+ * The complete dictionary token sequence is used when it contains at least + * {@code minimumTokenCount} tokens. Smaller resources are repeated in stable + * order until the minimum is reached. + *
+ * + * @param language bundled Radixor language + * @param minimumTokenCount minimum token count for timing + * @return token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createCorpus(final StemmerPatchTrieLoader.Language language, final int minimumTokenCount) + throws IOException { + Objects.requireNonNull(language, "language"); + if (minimumTokenCount < 1) { + throw new IllegalArgumentException("minimumTokenCount must be at least 1."); + } + if (minimumTokenCount == MINIMUM_TIMING_TOKEN_COUNT) { + return createCorpus(language); + } + + return buildTimingCorpus(language, minimumTokenCount); + } + + /** + * Creates or returns the shared complete corpus for a bundled language. + * + * @param language bundled Radixor language + * @return complete token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + static Corpus createFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + return cachedCorpus(FULL_CORPORA, language, false); + } + + /** + * Returns a cached corpus, creating it once per JVM when necessary. + * + * @param cache corpus cache + * @param language bundled Radixor language + * @param timing whether the timing-minimum corpus should be built + * @return cached corpus instance + * @throws IOException if the resource cannot be read + */ + private static Corpus cachedCorpus(final Map+ * This method is intended for exact-root quality accounting. It includes all + * single-token fields available in the dictionary resource and does not repeat + * small dictionaries to the timing minimum. + *
+ * + * @param language bundled Radixor language + * @return complete token corpus with expected roots + * @throws IOException if the resource cannot be read + */ + private static Corpus buildFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException { + Objects.requireNonNull(language, "language"); + + final List+ * Each benchmark operation processes the same changed-token dictionary corpus + * for one language, repeated only when the changed-token resource contains + * fewer than 5,000 token fields. The token corpus is built during trial setup + * from Radixor's bundled dictionary for that same language. Lucene TokenFilter + * methods include TokenStream and attribute overhead; direct Stempel measures + * the public table-driven stemmer API without TokenFilter overhead. + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@SuppressWarnings("deprecation") +public class MultiLanguageStemmerComparisonBenchmark { + + /** + * Shared language corpus and Radixor trie state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Czech benchmark state. + */ + private LanguageState czech; + + /** + * German benchmark state. + */ + private LanguageState german; + + /** + * Spanish benchmark state. + */ + private LanguageState spanish; + + /** + * Persian benchmark state. + */ + private LanguageState persian; + + /** + * Finnish benchmark state. + */ + private LanguageState finnish; + + /** + * French benchmark state. + */ + private LanguageState french; + + /** + * Hungarian benchmark state. + */ + private LanguageState hungarian; + + /** + * Italian benchmark state. + */ + private LanguageState italian; + + /** + * Norwegian Bokmal benchmark state. + */ + private LanguageState norwegianBokmal; + + /** + * Polish benchmark state. + */ + private LanguageState polish; + + /** + * Portuguese benchmark state. + */ + private LanguageState portuguese; + + /** + * Russian benchmark state. + */ + private LanguageState russian; + + /** + * Swedish benchmark state. + */ + private LanguageState swedish; + + /** + * Ukrainian benchmark state. + */ + private LanguageState ukrainian; + + /** + * Initializes all language resources before measurement. + * + * @throws IOException if a bundled language resource cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.czech = load(StemmerPatchTrieLoader.Language.CS_CZ); + this.german = load(StemmerPatchTrieLoader.Language.DE_DE); + this.spanish = load(StemmerPatchTrieLoader.Language.ES_ES); + this.persian = load(StemmerPatchTrieLoader.Language.FA_IR); + this.finnish = load(StemmerPatchTrieLoader.Language.FI_FI); + this.french = load(StemmerPatchTrieLoader.Language.FR_FR); + this.hungarian = load(StemmerPatchTrieLoader.Language.HU_HU); + this.italian = load(StemmerPatchTrieLoader.Language.IT_IT); + this.norwegianBokmal = load(StemmerPatchTrieLoader.Language.NB_NO); + this.polish = load(StemmerPatchTrieLoader.Language.PL_PL); + this.portuguese = load(StemmerPatchTrieLoader.Language.PT_PT); + this.russian = load(StemmerPatchTrieLoader.Language.RU_RU); + this.swedish = load(StemmerPatchTrieLoader.Language.SV_SE); + this.ukrainian = load(StemmerPatchTrieLoader.Language.UK_UA); + } + } + + /** + * Per-thread Lucene filter state. + */ + @State(Scope.Thread) + public static class LuceneFilterState { + + /** + * Czech stem filter. + */ + private final FilterPipeline czechStem = new FilterPipeline( + input -> new CzechStemFilter(lowercase(input))); + + /** + * German classic stem filter. + */ + private final FilterPipeline germanStem = new FilterPipeline( + input -> new GermanStemFilter(lowercase(input))); + + /** + * German light stem filter. + */ + private final FilterPipeline germanLightStem = new FilterPipeline( + input -> new GermanLightStemFilter(germanNormalize(input))); + + /** + * German minimal stem filter. + */ + private final FilterPipeline germanMinimalStem = new FilterPipeline( + input -> new GermanMinimalStemFilter(germanNormalize(input))); + + /** + * Spanish light stem filter. + */ + private final FilterPipeline spanishLightStem = new FilterPipeline( + input -> new SpanishLightStemFilter(lowercase(input))); + + /** + * Spanish minimal stem filter. + */ + private final FilterPipeline spanishMinimalStem = new FilterPipeline( + input -> new SpanishMinimalStemFilter(lowercase(input))); + + /** + * Spanish plural stem filter. + */ + private final FilterPipeline spanishPluralStem = new FilterPipeline( + input -> new SpanishPluralStemFilter(lowercase(input))); + + /** + * Persian stem filter. + */ + private final FilterPipeline persianStem = new FilterPipeline( + input -> new PersianStemFilter(persianNormalize(input))); + + /** + * Finnish light stem filter. + */ + private final FilterPipeline finnishLightStem = new FilterPipeline( + input -> new FinnishLightStemFilter(lowercase(input))); + + /** + * French light stem filter. + */ + private final FilterPipeline frenchLightStem = new FilterPipeline( + input -> new FrenchLightStemFilter(lowercase(input))); + + /** + * French minimal stem filter. + */ + private final FilterPipeline frenchMinimalStem = new FilterPipeline( + input -> new FrenchMinimalStemFilter(lowercase(input))); + + /** + * Hungarian light stem filter. + */ + private final FilterPipeline hungarianLightStem = new FilterPipeline( + input -> new HungarianLightStemFilter(lowercase(input))); + + /** + * Italian light stem filter. + */ + private final FilterPipeline italianLightStem = new FilterPipeline( + input -> new ItalianLightStemFilter(lowercase(input))); + + /** + * Norwegian light stem filter. + */ + private final FilterPipeline norwegianLightStem = new FilterPipeline( + input -> new NorwegianLightStemFilter(lowercase(input))); + + /** + * Norwegian minimal stem filter. + */ + private final FilterPipeline norwegianMinimalStem = new FilterPipeline( + input -> new NorwegianMinimalStemFilter(lowercase(input))); + + /** + * Polish Stempel token filter. + */ + private final FilterPipeline polishStempelStem = new FilterPipeline( + input -> new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()))); + + /** + * Polish Morfologik token filter. + */ + private final FilterPipeline polishMorfologik = new FilterPipeline(MorfologikFilter::new); + + /** + * Portuguese full stem filter. + */ + private final FilterPipeline portugueseStem = new FilterPipeline( + input -> new PortugueseStemFilter(lowercase(input))); + + /** + * Portuguese light stem filter. + */ + private final FilterPipeline portugueseLightStem = new FilterPipeline( + input -> new PortugueseLightStemFilter(lowercase(input))); + + /** + * Portuguese minimal stem filter. + */ + private final FilterPipeline portugueseMinimalStem = new FilterPipeline( + input -> new PortugueseMinimalStemFilter(lowercase(input))); + + /** + * Russian light stem filter. + */ + private final FilterPipeline russianLightStem = new FilterPipeline( + input -> new RussianLightStemFilter(lowercase(input))); + + /** + * Swedish light stem filter. + */ + private final FilterPipeline swedishLightStem = new FilterPipeline( + input -> new SwedishLightStemFilter(lowercase(input))); + + /** + * Swedish minimal stem filter. + */ + private final FilterPipeline swedishMinimalStem = new FilterPipeline( + input -> new SwedishMinimalStemFilter(lowercase(input))); + + /** + * Ukrainian Morfologik token filter. + */ + private FilterPipeline ukrainianMorfologik; + + /** + * Initializes filter state that needs benchmark-only dictionary resources. + * + * @throws IOException if a benchmark-only dictionary cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final Dictionary dictionary = loadUkrainianMorfologikDictionary(); + this.ukrainianMorfologik = new FilterPipeline(input -> new MorfologikFilter(input, dictionary)); + } + } + + /** + * Per-thread direct non-TokenFilter stemmer state. + */ + @State(Scope.Thread) + public static class DirectState { + + /** + * Direct Stempel stemmer using Lucene's default Polish table. + */ + private StempelStemmer polishStempelStemmer; + + /** + * Direct Ukrainian Morfologik dictionary lookup. + */ + private DictionaryLookup ukrainianMorfologikLookup; + + /** + * Initializes direct stemmer instances before measurement. + * + * @throws IOException if a benchmark-only dictionary cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.polishStempelStemmer = new StempelStemmer(PolishAnalyzer.getDefaultTable()); + this.ukrainianMorfologikLookup = new DictionaryLookup(loadUkrainianMorfologikDictionary()); + } + } + + /** + * Runs Radixor over the Czech corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void czechRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.czech, blackhole); + } + + /** + * Runs Lucene CzechStemFilter over the Czech corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void czechLuceneCzechStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.czechStem.run(sharedState.czech.tokens, blackhole); + } + + /** + * Runs Radixor over the German corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void germanRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.german, blackhole); + } + + /** + * Runs Lucene GermanStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Lucene GermanLightStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanLightStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Lucene GermanMinimalStemFilter over the German corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void germanLuceneGermanMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.germanMinimalStem.run(sharedState.german.tokens, blackhole); + } + + /** + * Runs Radixor over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void spanishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.spanish, blackhole); + } + + /** + * Runs Lucene SpanishLightStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishLightStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Lucene SpanishMinimalStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishMinimalStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Lucene SpanishPluralStemFilter over the Spanish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void spanishLuceneSpanishPluralStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.spanishPluralStem.run(sharedState.spanish.tokens, blackhole); + } + + /** + * Runs Radixor over the Persian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void persianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.persian, blackhole); + } + + /** + * Runs Lucene PersianStemFilter over the Persian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void persianLucenePersianStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.persianStem.run(sharedState.persian.tokens, blackhole); + } + + /** + * Runs Radixor over the Finnish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void finnishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.finnish, blackhole); + } + + /** + * Runs Lucene FinnishLightStemFilter over the Finnish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void finnishLuceneFinnishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.finnishLightStem.run(sharedState.finnish.tokens, blackhole); + } + + /** + * Runs Radixor over the French corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void frenchRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.french, blackhole); + } + + /** + * Runs Lucene FrenchLightStemFilter over the French corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void frenchLuceneFrenchLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.frenchLightStem.run(sharedState.french.tokens, blackhole); + } + + /** + * Runs Lucene FrenchMinimalStemFilter over the French corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void frenchLuceneFrenchMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.frenchMinimalStem.run(sharedState.french.tokens, blackhole); + } + + /** + * Runs Radixor over the Hungarian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void hungarianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.hungarian, blackhole); + } + + /** + * Runs Lucene HungarianLightStemFilter over the Hungarian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void hungarianLuceneHungarianLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.hungarianLightStem.run(sharedState.hungarian.tokens, blackhole); + } + + /** + * Runs Radixor over the Italian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void italianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.italian, blackhole); + } + + /** + * Runs Lucene ItalianLightStemFilter over the Italian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void italianLuceneItalianLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.italianLightStem.run(sharedState.italian.tokens, blackhole); + } + + /** + * Runs Radixor over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void norwegianBokmalRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.norwegianBokmal, blackhole); + } + + /** + * Runs Lucene NorwegianLightStemFilter over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void norwegianBokmalLuceneNorwegianLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.norwegianLightStem.run(sharedState.norwegianBokmal.tokens, blackhole); + } + + /** + * Runs Lucene NorwegianMinimalStemFilter over the Norwegian Bokmal corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void norwegianBokmalLuceneNorwegianMinimalStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.norwegianMinimalStem.run(sharedState.norwegianBokmal.tokens, blackhole); + } + + /** + * Runs Radixor over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void polishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.polish, blackhole); + } + + /** + * Runs Lucene Stempel direct API over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct stemmer state + * @param blackhole result sink + */ + @Benchmark + public void polishLuceneStempelStemmerDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.polish.tokens; + final StempelStemmer stemmer = directState.polishStempelStemmer; + for (String token : tokens) { + final StringBuilder stem = stemmer.stem(token); + blackhole.consume(stem == null ? token : stem.toString()); + } + } + + /** + * Runs Lucene StempelFilter over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void polishLuceneStempelFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.polishStempelStem.run(sharedState.polish.tokens, blackhole); + } + + /** + * Runs Lucene MorfologikFilter over the Polish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void polishLuceneMorfologikFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.polishMorfologik.run(sharedState.polish.tokens, blackhole); + } + + /** + * Runs Radixor over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void portugueseRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.portuguese, blackhole); + } + + /** + * Runs Lucene PortugueseStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Lucene PortugueseLightStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseLightStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseLightStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Lucene PortugueseMinimalStemFilter over the Portuguese corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void portugueseLucenePortugueseMinimalStemFilter(final SharedState sharedState, + final LuceneFilterState filterState, final Blackhole blackhole) throws IOException { + filterState.portugueseMinimalStem.run(sharedState.portuguese.tokens, blackhole); + } + + /** + * Runs Radixor over the Russian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void russianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.russian, blackhole); + } + + /** + * Runs Lucene RussianLightStemFilter over the Russian corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void russianLuceneRussianLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.russianLightStem.run(sharedState.russian.tokens, blackhole); + } + + /** + * Runs Radixor over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void swedishRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.swedish, blackhole); + } + + /** + * Runs Lucene SwedishLightStemFilter over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void swedishLuceneSwedishLightStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.swedishLightStem.run(sharedState.swedish.tokens, blackhole); + } + + /** + * Runs Lucene SwedishMinimalStemFilter over the Swedish corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void swedishLuceneSwedishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.swedishMinimalStem.run(sharedState.swedish.tokens, blackhole); + } + + /** + * Runs Radixor over the Ukrainian corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void ukrainianRadixor(final SharedState sharedState, final Blackhole blackhole) { + runRadixor(sharedState.ukrainian, blackhole); + } + + /** + * Runs direct Morfologik Ukrainian dictionary lookup over the Ukrainian corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct stemmer state + * @param blackhole result sink + */ + @Benchmark + public void ukrainianMorfologikDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.ukrainian.tokens; + final DictionaryLookup lookup = directState.ukrainianMorfologikLookup; + for (String token : tokens) { + blackhole.consume(firstMorfologikStem(token, lookup)); + } + } + + /** + * Runs Lucene MorfologikFilter with the Ukrainian dictionary over the Ukrainian + * corpus. + * + * @param sharedState shared benchmark state + * @param filterState reusable filter state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void ukrainianLuceneMorfologikFilter(final SharedState sharedState, final LuceneFilterState filterState, + final Blackhole blackhole) throws IOException { + filterState.ukrainianMorfologik.run(sharedState.ukrainian.tokens, blackhole); + } + + /** + * Loads one language benchmark state. + * + * @param language bundled Radixor language + * @return initialized language state + * @throws IOException if the corpus or trie cannot be loaded + */ + private static LanguageState load(final StemmerPatchTrieLoader.Language language) throws IOException { + final String[] tokens = LanguageBenchmarkCorpus.createTokens(language); + final FrequencyTrie+ * The benchmark intentionally rebinds the {@code String[]} corpus on every + * measured operation so the adaptation cost from the canonical string input to + * Lucene's mutable character attributes is included. + *
+ * + * @param tokens token corpus + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + private void run(final String[] tokens, final Blackhole blackhole) throws IOException { + this.input.setTokens(tokens); + this.output.reset(); + while (this.output.incrementToken()) { + blackhole.consume(this.termAttribute.toString()); + } + this.output.end(); + } + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java b/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java new file mode 100644 index 0000000..f63c4a7 --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.util.Objects; + +import org.egothor.stemmer.CompiledPatchCommand; +import org.egothor.stemmer.FrequencyTrie; + +/** + * Benchmark-only Radixor stemmer adapter for the canonical preferred-result + * path over normalized benchmark tokens. + * + *+ * The benchmark corpus is normalized during setup, so this adapter uses + * {@link FrequencyTrie#getNormalizedString(String)} to avoid measuring + * redundant lookup-time normalization. Patch commands are applied with the + * traversal direction persisted in the trie metadata. + *
+ * + *+ * Instances are mutable and intended for one JMH worker thread. + *
+ */ +final class RadixorBenchmarkStemmer { + + /** + * Compiled Radixor patch trie with decoded patch-command values. + */ + private final FrequencyTrie+ * Each benchmark operation processes the same changed-token Radixor + * dictionary-derived language corpus, repeated only when the changed-token + * resource contains fewer than 5,000 token fields. The direct Snowball method + * measures the isolated benchmark-only Snowball source. The Lucene + * SnowballFilter method measures Lucene's TokenStream integration path, + * including lower-case normalization and token attribute overhead. + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class SnowballLanguageStemmerComparisonBenchmark { + + /** + * Shared language corpus and Radixor trie state. + */ + @State(Scope.Benchmark) + public static class SharedState { + + /** + * Language/algorithm case under comparison. + */ + @Param({ "DANISH", "DUTCH", "FINNISH", "FRENCH", "GERMAN", "HUNGARIAN", "ITALIAN", + "NORWEGIAN_BOKMAL", "NORWEGIAN_NYNORSK", "PORTUGUESE", "RUSSIAN", "SPANISH", "SWEDISH", + "YIDDISH" }) + public String languageCaseName; + + /** + * Resolved language/algorithm case. + */ + private SnowballLanguageCase languageCase; + + /** + * Shared deterministic changed-token dictionary corpus. + */ + private String[] tokens; + + /** + * Compiled Radixor trie for the selected language. + */ + private RadixorBenchmarkStemmer radixorStemmer; + + /** + * Initializes shared language resources before measurement. + * + * @throws IOException if the corpus or trie cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.languageCase = SnowballLanguageCase.valueOf(this.languageCaseName); + this.tokens = LanguageBenchmarkCorpus.createTokens(this.languageCase.radixorLanguage()); + this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled( + this.languageCase.radixorLanguage(), true, + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + } + } + + /** + * Per-thread direct Snowball state. + */ + @State(Scope.Thread) + public static class DirectState { + + /** + * Reusable direct Snowball adapter. + */ + private SnowballStemmerAdapter snowballStemmer; + + /** + * Initializes direct Snowball state for the selected language. + * + * @param sharedState selected language state + */ + @Setup(Level.Trial) + public void setUp(final SharedState sharedState) { + this.snowballStemmer = sharedState.languageCase.createDirectStemmer(); + } + } + + /** + * Per-thread Lucene SnowballFilter state. + */ + @State(Scope.Thread) + public static class LuceneSnowballState { + + /** + * Reusable benchmark input stream. + */ + private BenchmarkTokenStream input; + + /** + * Reusable Lucene SnowballFilter output stream. + */ + private TokenStream output; + + /** + * Reusable term attribute. + */ + private CharTermAttribute termAttribute; + + /** + * Initializes Lucene SnowballFilter state for the selected language. + * + * @param sharedState selected language state + */ + @Setup(Level.Trial) + public void setUp(final SharedState sharedState) { + this.input = new BenchmarkTokenStream(new String[0]); + final TokenStream normalizedInput = new LowerCaseFilter(this.input); + this.output = new SnowballFilter(normalizedInput, sharedState.languageCase.luceneSnowballName()); + this.termAttribute = this.output.addAttribute(CharTermAttribute.class); + } + + /** + * Runs the reusable Lucene SnowballFilter over one corpus. + * + *+ * The benchmark intentionally rebinds the {@code String[]} corpus on every + * measured operation so the adaptation cost from the canonical string input to + * Lucene's mutable character attributes is included. + *
+ * + * @param tokens token corpus + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + void run(final String[] tokens, final Blackhole blackhole) throws IOException { + this.input.setTokens(tokens); + this.output.reset(); + while (this.output.incrementToken()) { + blackhole.consume(this.termAttribute.toString()); + } + this.output.end(); + } + } + + /** + * Runs Radixor over the selected Snowball-language corpus. + * + * @param sharedState shared benchmark state + * @param blackhole result sink + */ + @Benchmark + public void radixor(final SharedState sharedState, final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Runs the official Snowball direct Java implementation over the selected + * language corpus. + * + * @param sharedState shared benchmark state + * @param directState reusable direct Snowball state + * @param blackhole result sink + */ + @Benchmark + public void snowballDirect(final SharedState sharedState, final DirectState directState, + final Blackhole blackhole) { + final String[] tokens = sharedState.tokens; + final SnowballStemmerAdapter stemmer = directState.snowballStemmer; + + for (String token : tokens) { + blackhole.consume(stemmer.stem(token)); + } + } + + /** + * Runs Lucene SnowballFilter over the selected language corpus. + * + * @param sharedState shared benchmark state + * @param luceneState reusable Lucene Snowball state + * @param blackhole result sink + * @throws IOException if Lucene token streaming fails + */ + @Benchmark + public void luceneSnowballFilter(final SharedState sharedState, final LuceneSnowballState luceneState, + final Blackhole blackhole) throws IOException { + luceneState.run(sharedState.tokens, blackhole); + } +} diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java index 12bd12a..0044e03 100644 --- a/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java +++ b/src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java @@ -32,7 +32,7 @@ package org.egothor.stemmer.benchmark; import java.util.Objects; -import org.tartarus.snowball.SnowballStemmer; +import org.egothor.stemmer.benchmark.snowball.SnowballStemmer; /** * Small adapter around a Snowball stemmer instance used by benchmarks. diff --git a/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java new file mode 100644 index 0000000..f9978db --- /dev/null +++ b/src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java @@ -0,0 +1,813 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.egothor.stemmer.benchmark; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.ar.ArabicNormalizationFilter; +import org.apache.lucene.analysis.core.DecimalDigitFilter; +import org.apache.lucene.analysis.cz.CzechStemFilter; +import org.apache.lucene.analysis.de.GermanLightStemFilter; +import org.apache.lucene.analysis.de.GermanMinimalStemFilter; +import org.apache.lucene.analysis.de.GermanNormalizationFilter; +import org.apache.lucene.analysis.de.GermanStemFilter; +import org.apache.lucene.analysis.en.EnglishMinimalStemFilter; +import org.apache.lucene.analysis.en.EnglishPossessiveFilter; +import org.apache.lucene.analysis.en.KStemFilter; +import org.apache.lucene.analysis.en.PorterStemFilter; +import org.apache.lucene.analysis.es.SpanishLightStemFilter; +import org.apache.lucene.analysis.es.SpanishMinimalStemFilter; +import org.apache.lucene.analysis.es.SpanishPluralStemFilter; +import org.apache.lucene.analysis.fa.PersianNormalizationFilter; +import org.apache.lucene.analysis.fa.PersianStemFilter; +import org.apache.lucene.analysis.fi.FinnishLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchLightStemFilter; +import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter; +import org.apache.lucene.analysis.hu.HungarianLightStemFilter; +import org.apache.lucene.analysis.it.ItalianLightStemFilter; +import org.apache.lucene.analysis.morfologik.MorfologikFilter; +import org.apache.lucene.analysis.no.NorwegianLightStemFilter; +import org.apache.lucene.analysis.no.NorwegianMinimalStemFilter; +import org.apache.lucene.analysis.pl.PolishAnalyzer; +import org.apache.lucene.analysis.pt.PortugueseLightStemFilter; +import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter; +import org.apache.lucene.analysis.pt.PortugueseStemFilter; +import org.apache.lucene.analysis.ru.RussianLightStemFilter; +import org.apache.lucene.analysis.snowball.SnowballFilter; +import org.apache.lucene.analysis.stempel.StempelFilter; +import org.apache.lucene.analysis.stempel.StempelStemmer; +import org.apache.lucene.analysis.sv.SwedishLightStemFilter; +import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.egothor.stemmer.FrequencyTrie; +import org.egothor.stemmer.ReductionMode; +import org.egothor.stemmer.StemmerPatchTrieLoader; +import org.egothor.stemmer.benchmark.snowball.ext.englishStemmer; +import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import morfologik.stemming.Dictionary; +import morfologik.stemming.DictionaryLookup; +import morfologik.stemming.WordData; + +/** + * Emits exact-root agreement metrics through standard JMH result files. + * + *+ * This benchmark is a quality pass, not a throughput competitor. Each operation + * evaluates one stemmer against the complete Radixor dictionary resource for + * the matching language. The useful outputs are the JMH auxiliary counters + * {@code correctMatches}, {@code evaluatedTokens}, + * {@code changedCorrectMatches}, {@code changedEvaluatedTokens}, + * {@code rootPreservedMatches}, and {@code rootEvaluatedTokens}. + *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class StemmerComparisonBenchmarkQuality { + + /** + * Shared quality state for one candidate stemmer. + */ + @State(Scope.Benchmark) + public static class QualityState { + + /** + * Candidate stemmer whose exact-root agreement is measured. + */ + @Param({ + "ENGLISH_RADIXOR", + "ENGLISH_SNOWBALL_ORIGINAL_PORTER", + "ENGLISH_SNOWBALL_PORTER2", + "ENGLISH_LUCENE_PORTER_COPIED", + "ENGLISH_LUCENE_PORTER_FILTER", + "ENGLISH_LUCENE_KSTEM_FILTER", + "ENGLISH_LUCENE_MINIMAL_FILTER", + "ENGLISH_LUCENE_POSSESSIVE_FILTER", + "ENGLISH_PAICE_HUSK_LANCASTER", + "ENGLISH_OPENNLP_PORTER", + "CZECH_RADIXOR", + "CZECH_LUCENE_CZECH_STEM_FILTER", + "GERMAN_RADIXOR", + "GERMAN_LUCENE_GERMAN_STEM_FILTER", + "GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER", + "GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER", + "SPANISH_RADIXOR", + "SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER", + "SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER", + "SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER", + "PERSIAN_RADIXOR", + "PERSIAN_LUCENE_PERSIAN_STEM_FILTER", + "FINNISH_RADIXOR", + "FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER", + "FRENCH_RADIXOR", + "FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER", + "FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER", + "HUNGARIAN_RADIXOR", + "HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER", + "ITALIAN_RADIXOR", + "ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER", + "NORWEGIAN_BOKMAL_RADIXOR", + "NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER", + "NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER", + "POLISH_RADIXOR", + "POLISH_LUCENE_STEMPEL_DIRECT", + "POLISH_LUCENE_STEMPEL_FILTER", + "POLISH_LUCENE_MORFOLOGIK_FILTER", + "PORTUGUESE_RADIXOR", + "PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER", + "PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER", + "PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER", + "RUSSIAN_RADIXOR", + "RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER", + "SWEDISH_RADIXOR", + "SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER", + "SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER", + "UKRAINIAN_RADIXOR", + "UKRAINIAN_MORFOLOGIK_DIRECT", + "UKRAINIAN_LUCENE_MORFOLOGIK_FILTER", + "SNOWBALL_DANISH_RADIXOR", + "SNOWBALL_DANISH_DIRECT", + "SNOWBALL_DANISH_LUCENE_FILTER", + "SNOWBALL_DUTCH_RADIXOR", + "SNOWBALL_DUTCH_DIRECT", + "SNOWBALL_DUTCH_LUCENE_FILTER", + "SNOWBALL_FINNISH_RADIXOR", + "SNOWBALL_FINNISH_DIRECT", + "SNOWBALL_FINNISH_LUCENE_FILTER", + "SNOWBALL_FRENCH_RADIXOR", + "SNOWBALL_FRENCH_DIRECT", + "SNOWBALL_FRENCH_LUCENE_FILTER", + "SNOWBALL_GERMAN_RADIXOR", + "SNOWBALL_GERMAN_DIRECT", + "SNOWBALL_GERMAN_LUCENE_FILTER", + "SNOWBALL_HUNGARIAN_RADIXOR", + "SNOWBALL_HUNGARIAN_DIRECT", + "SNOWBALL_HUNGARIAN_LUCENE_FILTER", + "SNOWBALL_ITALIAN_RADIXOR", + "SNOWBALL_ITALIAN_DIRECT", + "SNOWBALL_ITALIAN_LUCENE_FILTER", + "SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR", + "SNOWBALL_NORWEGIAN_BOKMAL_DIRECT", + "SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER", + "SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR", + "SNOWBALL_NORWEGIAN_NYNORSK_DIRECT", + "SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER", + "SNOWBALL_PORTUGUESE_RADIXOR", + "SNOWBALL_PORTUGUESE_DIRECT", + "SNOWBALL_PORTUGUESE_LUCENE_FILTER", + "SNOWBALL_RUSSIAN_RADIXOR", + "SNOWBALL_RUSSIAN_DIRECT", + "SNOWBALL_RUSSIAN_LUCENE_FILTER", + "SNOWBALL_SPANISH_RADIXOR", + "SNOWBALL_SPANISH_DIRECT", + "SNOWBALL_SPANISH_LUCENE_FILTER", + "SNOWBALL_SWEDISH_RADIXOR", + "SNOWBALL_SWEDISH_DIRECT", + "SNOWBALL_SWEDISH_LUCENE_FILTER", + "SNOWBALL_YIDDISH_RADIXOR", + "SNOWBALL_YIDDISH_DIRECT", + "SNOWBALL_YIDDISH_LUCENE_FILTER" + }) + public String candidateName; + + /** + * Full dictionary corpus for the selected language. + */ + private LanguageBenchmarkCorpus.Corpus corpus; + + /** + * Candidate evaluator. + */ + private QualityEvaluator evaluator; + + /** + * Initializes corpus and evaluator before measurement. + * + * @throws IOException if dictionary or stemmer resources cannot be loaded + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + final QualityCandidate candidate = QualityCandidate.valueOf(this.candidateName); + this.corpus = LanguageBenchmarkCorpus.createFullCorpus(candidate.radixorLanguage()); + this.evaluator = candidate.createEvaluator(); + } + } + + /** + * JMH auxiliary counters for exact-root agreement. + */ + @State(Scope.Thread) + @AuxCounters(AuxCounters.Type.EVENTS) + public static class QualityCounters { + + /** + * Number of outputs equal to the dictionary root. + */ + public long correctMatches; + + /** + * Number of evaluated input tokens. + */ + public long evaluatedTokens; + + /** + * Number of exact-root matches where the input token differs from the + * expected root. + */ + public long changedCorrectMatches; + + /** + * Number of evaluated tokens where the input token differs from the expected + * root. + */ + public long changedEvaluatedTokens; + + /** + * Number of exact-root matches where the input token is already the expected + * root. + */ + public long rootPreservedMatches; + + /** + * Number of evaluated tokens where the input token is already the expected + * root. + */ + public long rootEvaluatedTokens; + + /** + * Resets counters before each measured iteration. + */ + @Setup(Level.Iteration) + public void reset() { + this.correctMatches = 0L; + this.evaluatedTokens = 0L; + this.changedCorrectMatches = 0L; + this.changedEvaluatedTokens = 0L; + this.rootPreservedMatches = 0L; + this.rootEvaluatedTokens = 0L; + } + } + + /** + * Runs exact-root agreement over the full dictionary corpus. + * + * @param state quality state + * @param counters auxiliary JMH counters + * @param blackhole result sink + * @return exact-root match count for this operation + * @throws IOException if Lucene streaming fails + */ + @Benchmark + public int exactRootAgreement(final QualityState state, final QualityCounters counters, final Blackhole blackhole) + throws IOException { + final QualityResult result = state.evaluator.evaluate(state.corpus, blackhole); + counters.correctMatches += result.correctMatches(); + counters.evaluatedTokens += result.evaluatedTokens(); + counters.changedCorrectMatches += result.changedCorrectMatches(); + counters.changedEvaluatedTokens += result.changedEvaluatedTokens(); + counters.rootPreservedMatches += result.rootPreservedMatches(); + counters.rootEvaluatedTokens += result.rootEvaluatedTokens(); + return result.correctMatches(); + } + + /** + * Candidate stemmers that can be evaluated against a Radixor resource. + */ + private enum QualityCandidate { + ENGLISH_RADIXOR(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_SNOWBALL_ORIGINAL_PORTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_SNOWBALL_PORTER2(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_PORTER_COPIED(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_PORTER_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_KSTEM_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_MINIMAL_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_LUCENE_POSSESSIVE_FILTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_PAICE_HUSK_LANCASTER(StemmerPatchTrieLoader.Language.US_UK), + ENGLISH_OPENNLP_PORTER(StemmerPatchTrieLoader.Language.US_UK), + CZECH_RADIXOR(StemmerPatchTrieLoader.Language.CS_CZ), + CZECH_LUCENE_CZECH_STEM_FILTER(StemmerPatchTrieLoader.Language.CS_CZ), + GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE), + SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES), + PERSIAN_RADIXOR(StemmerPatchTrieLoader.Language.FA_IR), + PERSIAN_LUCENE_PERSIAN_STEM_FILTER(StemmerPatchTrieLoader.Language.FA_IR), + FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI), + FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FI_FI), + FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR), + FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR), + FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR), + HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU), + HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.HU_HU), + ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT), + ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.IT_IT), + NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO), + NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO), + NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO), + POLISH_RADIXOR(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_STEMPEL_DIRECT(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_STEMPEL_FILTER(StemmerPatchTrieLoader.Language.PL_PL), + POLISH_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.PL_PL), + PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT), + RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU), + RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.RU_RU), + SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE), + SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE), + SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE), + UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA), + UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA), + UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA), + SNOWBALL_DANISH_RADIXOR(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH), + SNOWBALL_DUTCH_RADIXOR(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_DUTCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH), + SNOWBALL_FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FINNISH_DIRECT(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FINNISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH), + SNOWBALL_FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_FRENCH_DIRECT(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_FRENCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH), + SNOWBALL_GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_GERMAN_DIRECT(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_GERMAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN), + SNOWBALL_HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_HUNGARIAN_DIRECT(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_HUNGARIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN), + SNOWBALL_ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_ITALIAN_DIRECT(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_ITALIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN), + SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_BOKMAL_DIRECT(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NB_NO, + SnowballLanguageCase.NORWEGIAN_BOKMAL), + SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_NORWEGIAN_NYNORSK_DIRECT(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO, + SnowballLanguageCase.NORWEGIAN_NYNORSK), + SNOWBALL_PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE), + SNOWBALL_RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_RUSSIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN), + SNOWBALL_SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SPANISH_DIRECT(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SPANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH), + SNOWBALL_SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_SWEDISH_DIRECT(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_SWEDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH), + SNOWBALL_YIDDISH_RADIXOR(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH), + SNOWBALL_YIDDISH_DIRECT(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH), + SNOWBALL_YIDDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH); + + /** + * Radixor dictionary language used as truth. + */ + private final StemmerPatchTrieLoader.Language radixorLanguage; + + /** + * Optional Snowball language mapping. + */ + private final SnowballLanguageCase snowballLanguageCase; + + /** + * Creates a candidate. + * + * @param radixorLanguage Radixor dictionary language + */ + QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage) { + this(radixorLanguage, null); + } + + /** + * Creates a candidate. + * + * @param radixorLanguage Radixor dictionary language + * @param snowballLanguageCase matching Snowball case + */ + QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage, + final SnowballLanguageCase snowballLanguageCase) { + this.radixorLanguage = radixorLanguage; + this.snowballLanguageCase = snowballLanguageCase; + } + + /** + * Returns the Radixor dictionary language. + * + * @return Radixor language + */ + StemmerPatchTrieLoader.Language radixorLanguage() { + return this.radixorLanguage; + } + + /** + * Creates the evaluator for this candidate. + * + * @return quality evaluator + * @throws IOException if stemmer resources cannot be loaded + */ + QualityEvaluator createEvaluator() throws IOException { + if (name().endsWith("_RADIXOR")) { + return direct(createRadixorStemmer(this.radixorLanguage)); + } + if (name().endsWith("_DIRECT") && this.snowballLanguageCase != null) { + return direct(this.snowballLanguageCase.createDirectStemmer()::stem); + } + if (name().endsWith("_LUCENE_FILTER") && this.snowballLanguageCase != null) { + return tokenFilter(input -> new SnowballFilter(new LowerCaseFilter(input), + this.snowballLanguageCase.luceneSnowballName())); + } + + return switch (this) { + case ENGLISH_SNOWBALL_ORIGINAL_PORTER -> direct(new SnowballStemmerAdapter(porterStemmer::new)::stem); + case ENGLISH_SNOWBALL_PORTER2 -> direct(new SnowballStemmerAdapter(englishStemmer::new)::stem); + case ENGLISH_LUCENE_PORTER_COPIED -> direct(new LucenePorterStemmerCopied()::stem); + case ENGLISH_LUCENE_PORTER_FILTER -> tokenFilter(PorterStemFilter::new); + case ENGLISH_LUCENE_KSTEM_FILTER -> tokenFilter(KStemFilter::new); + case ENGLISH_LUCENE_MINIMAL_FILTER -> tokenFilter(EnglishMinimalStemFilter::new); + case ENGLISH_LUCENE_POSSESSIVE_FILTER -> tokenFilter(EnglishPossessiveFilter::new); + case ENGLISH_PAICE_HUSK_LANCASTER -> direct(new PaiceHuskLancasterStemmer()::stem); + case ENGLISH_OPENNLP_PORTER -> { + final opennlp.tools.stemmer.PorterStemmer stemmer = + new opennlp.tools.stemmer.PorterStemmer(); + yield direct(token -> stemmer.stem(token).toString()); + } + case CZECH_LUCENE_CZECH_STEM_FILTER -> tokenFilter(input -> new CzechStemFilter(lowercase(input))); + case GERMAN_LUCENE_GERMAN_STEM_FILTER -> tokenFilter(input -> new GermanStemFilter(lowercase(input))); + case GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new GermanLightStemFilter(germanNormalize(input))); + case GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new GermanMinimalStemFilter(germanNormalize(input))); + case SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new SpanishLightStemFilter(lowercase(input))); + case SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new SpanishMinimalStemFilter(lowercase(input))); + case SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER -> + tokenFilter(input -> new SpanishPluralStemFilter(lowercase(input))); + case PERSIAN_LUCENE_PERSIAN_STEM_FILTER -> + tokenFilter(input -> new PersianStemFilter(persianNormalize(input))); + case FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new FinnishLightStemFilter(lowercase(input))); + case FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new FrenchLightStemFilter(lowercase(input))); + case FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new FrenchMinimalStemFilter(lowercase(input))); + case HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new HungarianLightStemFilter(lowercase(input))); + case ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new ItalianLightStemFilter(lowercase(input))); + case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new NorwegianLightStemFilter(lowercase(input))); + case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new NorwegianMinimalStemFilter(lowercase(input))); + case POLISH_LUCENE_STEMPEL_DIRECT -> { + final StempelStemmer stemmer = new StempelStemmer(PolishAnalyzer.getDefaultTable()); + yield direct(token -> { + final StringBuilder stem = stemmer.stem(token); + return stem == null ? token : stem.toString(); + }); + } + case POLISH_LUCENE_STEMPEL_FILTER -> + tokenFilter(input -> new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable()))); + case POLISH_LUCENE_MORFOLOGIK_FILTER -> tokenFilter(MorfologikFilter::new); + case PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER -> + tokenFilter(input -> new PortugueseStemFilter(lowercase(input))); + case PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER -> + tokenFilter(input -> new PortugueseLightStemFilter(lowercase(input))); + case PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new PortugueseMinimalStemFilter(lowercase(input))); + case RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER -> + tokenFilter(input -> new RussianLightStemFilter(lowercase(input))); + case SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER -> + tokenFilter(input -> new SwedishLightStemFilter(lowercase(input))); + case SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER -> + tokenFilter(input -> new SwedishMinimalStemFilter(lowercase(input))); + case UKRAINIAN_MORFOLOGIK_DIRECT -> { + final DictionaryLookup lookup = new DictionaryLookup(loadUkrainianMorfologikDictionary()); + yield direct(token -> firstMorfologikStem(token, lookup)); + } + case UKRAINIAN_LUCENE_MORFOLOGIK_FILTER -> { + final Dictionary dictionary = loadUkrainianMorfologikDictionary(); + yield tokenFilter(input -> new MorfologikFilter(input, dictionary)); + } + default -> throw new IllegalStateException("No evaluator for " + this + "."); + }; + } + } + + /** + * Direct stemmer function. + */ + @FunctionalInterface + private interface Stemmer { + + /** + * Produces one stem. + * + * @param token input token + * @return produced stem + */ + String stem(String token); + } + + /** + * Quality evaluator for one candidate. + */ + @FunctionalInterface + private interface QualityEvaluator { + + /** + * Evaluates exact-root agreement for one corpus. + * + * @param corpus token/root corpus + * @param blackhole result sink + * @return exact-root match count + * @throws IOException if Lucene streaming fails + */ + QualityResult evaluate(LanguageBenchmarkCorpus.Corpus corpus, Blackhole blackhole) throws IOException; + } + + /** + * Exact-root agreement counters for one quality operation. + * + * @param correctMatches total exact-root matches + * @param evaluatedTokens total evaluated tokens + * @param changedCorrectMatches exact-root matches where token differs from root + * @param changedEvaluatedTokens evaluated tokens where token differs from root + * @param rootPreservedMatches exact-root matches where token already equals root + * @param rootEvaluatedTokens evaluated tokens where token already equals root + */ + private record QualityResult(int correctMatches, int evaluatedTokens, int changedCorrectMatches, + int changedEvaluatedTokens, int rootPreservedMatches, int rootEvaluatedTokens) { + } + + /** + * Creates a direct evaluator. + * + * @param stemmer direct stemmer + * @return quality evaluator + */ + private static QualityEvaluator direct(final Stemmer stemmer) { + Objects.requireNonNull(stemmer, "stemmer"); + return (corpus, blackhole) -> { + int correct = 0; + int changedCorrect = 0; + int changedEvaluated = 0; + int rootPreserved = 0; + int rootEvaluated = 0; + final String[] tokens = corpus.tokens(); + final String[] expectedRoots = corpus.expectedRoots(); + for (int index = 0; index < tokens.length; index++) { + final String token = tokens[index]; + final String expectedRoot = expectedRoots[index]; + final String actual = stemmer.stem(token); + blackhole.consume(actual); + final boolean exact = Objects.equals(expectedRoot, actual); + if (exact) { + correct++; + } + if (Objects.equals(token, expectedRoot)) { + rootEvaluated++; + if (exact) { + rootPreserved++; + } + } else { + changedEvaluated++; + if (exact) { + changedCorrect++; + } + } + } + return new QualityResult(correct, tokens.length, changedCorrect, changedEvaluated, rootPreserved, + rootEvaluated); + }; + } + + /** + * Creates a TokenFilter evaluator. + * + * @param factory token stream factory + * @return quality evaluator + */ + private static QualityEvaluator tokenFilter(final Function+ * Compilation selects a concrete command class for the patch shape. Common + * one-operation commands such as suffix deletion, prefix deletion, character + * append, character prepend, and single-character replacement therefore execute + * without a per-application opcode switch. Multi-operation patches are represented + * as a compound command containing concrete atomic operations. + *
+ * + *+ * Instances are immutable and thread-safe. Setup code may cache and share them + * freely across tries and benchmark states. + *
+ */ +@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.UseVarargs" }) +public abstract class CompiledPatchCommand { + + /** + * Return value used when the caller-owned output range is too small. + */ + public static final int APPLY_INSUFFICIENT_CAPACITY = PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY; + + /** + * Opcode for deleting one or more characters. + */ + private static final char DELETE_OPCODE = 'D'; + + /** + * Opcode for inserting one character. + */ + private static final char INSERT_OPCODE = 'I'; + + /** + * Opcode for replacing one character. + */ + private static final char REPLACE_OPCODE = 'R'; + + /** + * Opcode for skipping one or more unchanged characters. + */ + private static final char SKIP_OPCODE = '-'; + + /** + * Opcode for a canonical no-operation patch. + */ + private static final char NOOP_OPCODE = 'N'; + + /** + * Canonical no-operation patch argument. + */ + private static final char NOOP_ARGUMENT = 'a'; + + /** + * Serialized length of one opcode/argument patch command. + */ + private static final int SINGLE_COMMAND_LENGTH = 2; + + /** + * Smallest decoded skip/delete count accepted by the patch format. + */ + private static final int MINIMUM_COUNT = 1; + + /** + * First encoded count argument. + */ + private static final char FIRST_COUNT_ARGUMENT = 'a'; + + /** + * Prefix used in unsupported NOOP patch argument exceptions. + */ + private static final String MSG_NOOP = "Unsupported NOOP patch argument: "; + + /** + * Prefix used in unsupported patch opcode exceptions. + */ + private static final String MSG_OPCODE = "Unsupported patch opcode: "; + + /** + * Traversal direction used by this command. + */ + private final WordTraversalDirection traversalDirection; + + /** + * Constant result-length delta applied by this command. + */ + private final int lengthDelta; + + /** + * Minimum source length required before this command can be applied. + */ + private final int minimumSourceLength; + + /** + * Creates one compiled command. + * + * @param traversalDirection traversal direction used by this command + * @param lengthDelta constant result-length delta for this command + * @param minimumSourceLength minimum source length required for application + */ + protected CompiledPatchCommand(final WordTraversalDirection traversalDirection, final int lengthDelta, + final int minimumSourceLength) { + this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection"); + this.lengthDelta = lengthDelta; + this.minimumSourceLength = minimumSourceLength; + } + + /** + * Creates a builder that compiles one serialized patch command. + * + * @param patchCommand serialized patch command, or {@code null} for a + * preserve-only command + * @param traversalDirection traversal direction used by the patch command + * @return builder configured for the supplied command + * @throws NullPointerException if {@code traversalDirection} is {@code null} + */ + public static Builder builder(final String patchCommand, final WordTraversalDirection traversalDirection) { + return new Builder(patchCommand, traversalDirection); + } + + /** + * Compiles a serialized patch command for repeated application. + * + * @param patchCommand serialized patch command, or {@code null} for a + * preserve-only command + * @param traversalDirection traversal direction used by the patch command + * @return immutable compiled patch command + * @throws NullPointerException if {@code traversalDirection} is + * {@code null} + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public static CompiledPatchCommand compile(final String patchCommand, + final WordTraversalDirection traversalDirection) { + return builder(patchCommand, traversalDirection).build(); + } + + /** + * Applies this command to one source word and returns the transformed word. + * + * @param source source word + * @return transformed word, or {@code null} when {@code source} is + * {@code null} + */ + public final String apply(final String source) { + if (source == null) { + return null; + } + return applyNonNull(source); + } + + /** + * Applies this command from a character sequence into caller-owned output + * storage. + * + * @param source source text + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if the output range is invalid + */ + public final int applyTo(final CharSequence source, final char[] output, final int outputOffset, + final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + return applyTo(source, 0, source.length(), output, outputOffset, outputLength); + } + + /** + * Applies this command from a character-sequence slice into caller-owned output + * storage. + * + * @param source source text + * @param sourceOffset first source offset + * @param sourceLength number of source characters + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if any range is invalid + */ + public final int applyTo(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length()); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + + final int producedLength = computeAppliedLength(sourceLength); + if (producedLength > outputLength) { + return APPLY_INSUFFICIENT_CAPACITY; + } + applySequenceToOutput(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + return producedLength; + } + + /** + * Applies this command from a character-array slice into caller-owned output + * storage. + * + * @param source source storage + * @param sourceOffset first source offset + * @param sourceLength number of source characters + * @param output output storage + * @param outputOffset first writable output offset + * @param outputLength writable output capacity + * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} + * when {@code outputLength} is too small + * @throws NullPointerException if {@code source} or {@code output} is + * {@code null} + * @throws IndexOutOfBoundsException if any range is invalid + * @throws IllegalArgumentException if source and output ranges overlap in the + * same array + */ + public final int applyTo(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int outputLength) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(output, "output"); + Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length); + Objects.checkFromIndexSize(outputOffset, outputLength, output.length); + validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength); + + final int producedLength = computeAppliedLength(sourceLength); + if (producedLength > outputLength) { + return APPLY_INSUFFICIENT_CAPACITY; + } + applyArrayToOutput(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + return producedLength; + } + + /** + * Returns this command traversal direction. + * + * @return traversal direction + */ + public final WordTraversalDirection traversalDirection() { + return this.traversalDirection; + } + + /** + * Reports whether this command preserves every non-null source unchanged. + * + *+ * Hot paths can use this method to avoid output-buffer copying and result-string + * allocation for canonical no-operation dictionary entries. + *
+ * + * @return {@code true} when {@link #apply(String)} always returns the supplied + * source reference for non-null input + */ + public abstract boolean preservesAllSources(); + + /** + * Applies this command to a non-null source string. + * + * @param source non-null source word + * @return transformed word + */ + protected abstract String applyNonNull(String source); + + /** + * Computes the output length for a source of the supplied length. + * + * @param sourceLength source length + * @return output length + */ + protected final int computeAppliedLength(final int sourceLength) { + if (sourceLength < this.minimumSourceLength) { + return sourceLength; + } + final int appliedLength = sourceLength + this.lengthDelta; + return appliedLength < MINIMUM_COUNT ? sourceLength : appliedLength; + } + + /** + * Returns whether this command can produce a non-empty result for the supplied + * source length. + * + * @param sourceLength source length + * @return {@code true} when the constant command delta keeps the result + * non-empty + */ + protected final boolean hasApplicableLength(final int sourceLength) { + return sourceLength >= this.minimumSourceLength && sourceLength + this.lengthDelta >= MINIMUM_COUNT; + } + + /** + * Applies this command from a character sequence into caller-owned output. + * + * @param source source text + * @param sourceOffset first source offset + * @param sourceLength source length + * @param output output storage + * @param outputOffset first output offset + * @param producedLength computed produced length + */ + protected abstract void applySequenceToOutput(CharSequence source, int sourceOffset, int sourceLength, + char[] output, int outputOffset, int producedLength); + + /** + * Applies this command from a character array into caller-owned output. + * + * @param source source storage + * @param sourceOffset first source offset + * @param sourceLength source length + * @param output output storage + * @param outputOffset first output offset + * @param producedLength computed produced length + */ + protected abstract void applyArrayToOutput(char[] source, int sourceOffset, int sourceLength, + char[] output, int outputOffset, int producedLength); + + /** + * Builder that compiles one serialized patch command to the most specific + * runtime command class available. + */ + public static final class Builder { + + /** + * Serialized patch command. + */ + private final String patchCommand; + + /** + * Traversal direction for the command. + */ + private final WordTraversalDirection traversalDirection; + + private Builder(final String patchCommand, final WordTraversalDirection traversalDirection) { + this.patchCommand = patchCommand; + this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection"); + } + + /** + * Builds the concrete command instance. + * + * @return compiled command instance + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP + * argument + */ + public CompiledPatchCommand build() { + if (this.patchCommand == null) { + return preserve(this.traversalDirection); + } + + final int patchLength = this.patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { + return preserve(this.traversalDirection); + } + if (patchLength == SINGLE_COMMAND_LENGTH) { + return compileSingle(this.patchCommand.charAt(0), this.patchCommand.charAt(1), + this.traversalDirection); + } + + final int operationCount = patchLength >> 1; + final char[] opcodes = new char[operationCount]; + final int[] operands = new int[operationCount]; + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += SINGLE_COMMAND_LENGTH) { + final int operationIndex = patchIndex >> 1; + final char opcode = this.patchCommand.charAt(patchIndex); + final int operand = compileOperand(opcode, this.patchCommand.charAt(patchIndex + 1)); + if (operand < 0) { + return preserve(this.traversalDirection); + } + opcodes[operationIndex] = opcode; + operands[operationIndex] = operand; + } + + final int lengthDelta = computeLengthDelta(opcodes, operands); + final int minimumSourceLength = this.traversalDirection == WordTraversalDirection.BACKWARD + ? computeBackwardMinimumSourceLength(opcodes, operands) + : computeForwardMinimumSourceLength(opcodes, operands); + return this.traversalDirection == WordTraversalDirection.BACKWARD + ? new BackwardCompoundCommand(this.traversalDirection, opcodes, operands, lengthDelta, + minimumSourceLength) + : new ForwardCompoundCommand(this.traversalDirection, opcodes, operands, lengthDelta, + minimumSourceLength); + } + } + + private static CompiledPatchCommand preserve(final WordTraversalDirection traversalDirection) { + return new PreserveCommand(traversalDirection); + } + + private static CompiledPatchCommand compileSingle(final char opcode, final char argument, + final WordTraversalDirection traversalDirection) { + switch (opcode) { + case DELETE_OPCODE: + final int deleteCount = decodeEncodedCount(argument); + if (deleteCount < MINIMUM_COUNT) { + return preserve(traversalDirection); + } + return traversalDirection == WordTraversalDirection.BACKWARD + ? new DeleteSuffixCommand(traversalDirection, deleteCount) + : new DeletePrefixCommand(traversalDirection, deleteCount); + case INSERT_OPCODE: + return traversalDirection == WordTraversalDirection.BACKWARD + ? new AppendCharacterCommand(traversalDirection, argument) + : new PrependCharacterCommand(traversalDirection, argument); + case REPLACE_OPCODE: + return traversalDirection == WordTraversalDirection.BACKWARD + ? new ReplaceLastCharacterCommand(traversalDirection, argument) + : new ReplaceFirstCharacterCommand(traversalDirection, argument); + case SKIP_OPCODE: + return preserve(traversalDirection); + case NOOP_OPCODE: + if (argument != NOOP_ARGUMENT) { + throw new IllegalArgumentException(MSG_NOOP + argument); + } + return preserve(traversalDirection); + default: + throw new IllegalArgumentException(MSG_OPCODE + opcode); + } + } + + private static int compileOperand(final char opcode, final char argument) { + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = decodeEncodedCount(argument); + return skipCount < MINIMUM_COUNT ? -1 : skipCount; + case DELETE_OPCODE: + final int deleteCount = decodeEncodedCount(argument); + return deleteCount < MINIMUM_COUNT ? -1 : deleteCount; + case INSERT_OPCODE: + case REPLACE_OPCODE: + return argument; + case NOOP_OPCODE: + if (argument != NOOP_ARGUMENT) { + throw new IllegalArgumentException(MSG_NOOP + argument); + } + return -1; + default: + throw new IllegalArgumentException(MSG_OPCODE + opcode); + } + } + + private static int computeLengthDelta(final char[] opcodes, final int[] operands) { + int lengthDelta = 0; + for (int index = 0; index < opcodes.length; index++) { + switch (opcodes[index]) { + case DELETE_OPCODE: + lengthDelta -= operands[index]; + break; + case INSERT_OPCODE: + lengthDelta++; + break; + case SKIP_OPCODE: + case REPLACE_OPCODE: + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return lengthDelta; + } + + private static int computeForwardMinimumSourceLength(final char[] opcodes, final int[] operands) { + int minimumSourceLength = 0; + int position = 0; + int lengthDelta = 0; + for (int index = 0; index < opcodes.length; index++) { + final int operand = operands[index]; + switch (opcodes[index]) { + case SKIP_OPCODE: + position += operand; + break; + case DELETE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position + operand - lengthDelta); + lengthDelta -= operand; + break; + case INSERT_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position - lengthDelta); + lengthDelta++; + position++; + break; + case REPLACE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, position + 1 - lengthDelta); + position++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return minimumSourceLength; + } + + private static int computeBackwardMinimumSourceLength(final char[] opcodes, final int[] operands) { + int minimumSourceLength = 0; + int consumedFromEnd = 0; + for (int index = 0; index < opcodes.length; index++) { + final int operand = operands[index]; + switch (opcodes[index]) { + case SKIP_OPCODE: + consumedFromEnd += operand; + break; + case DELETE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd + operand); + consumedFromEnd += operand; + break; + case INSERT_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd); + break; + case REPLACE_OPCODE: + minimumSourceLength = Math.max(minimumSourceLength, consumedFromEnd + 1); + consumedFromEnd++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcodes[index]); + } + } + return minimumSourceLength; + } + + private static int decodeEncodedCount(final char argument) { + if (argument < FIRST_COUNT_ARGUMENT) { + return -1; + } + return argument - FIRST_COUNT_ARGUMENT + MINIMUM_COUNT; + } + + private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset) { + if (sourceLength <= 0) { + return; + } + if (source instanceof String sourceString) { + sourceString.getChars(sourceOffset, sourceOffset + sourceLength, output, outputOffset); + return; + } + for (int index = 0; index < sourceLength; index++) { + output[outputOffset + index] = source.charAt(sourceOffset + index); + } + } + + private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int outputLength) { + if (!sameArray(source, output) || sourceLength == 0 || outputLength == 0) { + return; + } + final int sourceEnd = sourceOffset + sourceLength; + final int outputEnd = outputOffset + outputLength; + if (sourceOffset < outputEnd && outputOffset < sourceEnd) { + throw new IllegalArgumentException("source and output ranges must not overlap."); + } + } + + @SuppressWarnings("PMD.CompareObjectsWithEquals") + private static boolean sameArray(final char[] left, final char[] right) { + return left == right; + } + + /** + * Command that preserves the source unchanged. + */ + private static final class PreserveCommand extends CompiledPatchCommand { + + private PreserveCommand(final WordTraversalDirection traversalDirection) { + super(traversalDirection, 0, 0); + } + + @Override + protected String applyNonNull(final String source) { + return source; + } + + @Override + public boolean preservesAllSources() { + return true; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + + /** + * Command that deletes characters from the logical suffix of a backward patch. + */ + private static final class DeleteSuffixCommand extends CompiledPatchCommand { + + /** + * Number of suffix characters deleted. + */ + private final int count; + + private DeleteSuffixCommand(final WordTraversalDirection traversalDirection, final int count) { + super(traversalDirection, -count, 0); + this.count = count; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + return source.substring(0, sourceLength - this.count); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, producedLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, producedLength); + } + } + + /** + * Command that deletes characters from the logical prefix of a forward patch. + */ + private static final class DeletePrefixCommand extends CompiledPatchCommand { + + /** + * Number of prefix characters deleted. + */ + private final int count; + + private DeletePrefixCommand(final WordTraversalDirection traversalDirection, final int count) { + super(traversalDirection, -count, 0); + this.count = count; + } + + @Override + protected String applyNonNull(final String source) { + if (!hasApplicableLength(source.length())) { + return source; + } + return source.substring(this.count); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + final int effectiveOffset = producedLength == sourceLength ? sourceOffset : sourceOffset + this.count; + copySource(source, effectiveOffset, producedLength, output, outputOffset); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + final int effectiveOffset = producedLength == sourceLength ? sourceOffset : sourceOffset + this.count; + System.arraycopy(source, effectiveOffset, output, outputOffset, producedLength); + } + } + + /** + * Command that appends one character to a backward patch result. + */ + private static final class AppendCharacterCommand extends CompiledPatchCommand { + + /** + * Appended character. + */ + private final char character; + + private AppendCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 1, 0); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + final char[] target = new char[sourceLength + 1]; + source.getChars(0, sourceLength, target, 0); + target[sourceLength] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + output[outputOffset + sourceLength] = this.character; + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + output[outputOffset + sourceLength] = this.character; + } + } + + /** + * Command that prepends one character to a forward patch result. + */ + private static final class PrependCharacterCommand extends CompiledPatchCommand { + + /** + * Prepended character. + */ + private final char character; + + private PrependCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 1, 0); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + final char[] target = new char[sourceLength + 1]; + target[0] = this.character; + source.getChars(0, sourceLength, target, 1); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + output[outputOffset] = this.character; + copySource(source, sourceOffset, sourceLength, output, outputOffset + 1); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + output[outputOffset] = this.character; + System.arraycopy(source, sourceOffset, output, outputOffset + 1, sourceLength); + } + } + + /** + * Command that replaces the final character of a backward patch result. + */ + private static final class ReplaceLastCharacterCommand extends CompiledPatchCommand { + + /** + * Replacement character. + */ + private final char character; + + private ReplaceLastCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 0, 1); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (sourceLength == 0) { + return source; + } + final char[] target = source.toCharArray(); + target[sourceLength - 1] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + if (sourceLength > 0) { + output[outputOffset + sourceLength - 1] = this.character; + } + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + if (sourceLength > 0) { + output[outputOffset + sourceLength - 1] = this.character; + } + } + } + + /** + * Command that replaces the first character of a forward patch result. + */ + private static final class ReplaceFirstCharacterCommand extends CompiledPatchCommand { + + /** + * Replacement character. + */ + private final char character; + + private ReplaceFirstCharacterCommand(final WordTraversalDirection traversalDirection, final char character) { + super(traversalDirection, 0, 1); + this.character = character; + } + + @Override + protected String applyNonNull(final String source) { + if (source.isEmpty()) { + return source; + } + final char[] target = source.toCharArray(); + target[0] = this.character; + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + if (sourceLength > 0) { + output[outputOffset] = this.character; + } + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + if (sourceLength > 0) { + output[outputOffset] = this.character; + } + } + } + + /** + * Compound command that applies atomic operations in backward traversal order. + */ + private static final class BackwardCompoundCommand extends CompiledPatchCommand { + + /** + * Operation opcodes in serialized order. + */ + private final char[] opcodes; + + /** + * Operation counts or character operands in serialized order. + */ + private final int[] operands; + + private BackwardCompoundCommand(final WordTraversalDirection traversalDirection, final char[] opcodes, + final int[] operands, final int lengthDelta, final int minimumSourceLength) { + super(traversalDirection, lengthDelta, minimumSourceLength); + this.opcodes = opcodes; + this.operands = operands; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + final int producedLength = computeAppliedLength(sourceLength); + final char[] target = new char[producedLength]; + writeSequence(source, 0, sourceLength, target, 0, producedLength); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + writeSequence(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + writeArray(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + private void writeSequence(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeBackwardSequence(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + } + + private void writeArray(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeBackwardArray(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + } + + /** + * Compound command that applies atomic operations in forward traversal order. + */ + private static final class ForwardCompoundCommand extends CompiledPatchCommand { + + /** + * Operation opcodes in serialized order. + */ + private final char[] opcodes; + + /** + * Operation counts or character operands in serialized order. + */ + private final int[] operands; + + private ForwardCompoundCommand(final WordTraversalDirection traversalDirection, final char[] opcodes, + final int[] operands, final int lengthDelta, final int minimumSourceLength) { + super(traversalDirection, lengthDelta, minimumSourceLength); + this.opcodes = opcodes; + this.operands = operands; + } + + @Override + protected String applyNonNull(final String source) { + final int sourceLength = source.length(); + if (!hasApplicableLength(sourceLength)) { + return source; + } + final int producedLength = computeAppliedLength(sourceLength); + final char[] target = new char[producedLength]; + writeSequence(source, 0, sourceLength, target, 0, producedLength); + return new String(target); + } + + @Override + public boolean preservesAllSources() { + return false; + } + + @Override + protected void applySequenceToOutput(final CharSequence source, final int sourceOffset, + final int sourceLength, final char[] output, final int outputOffset, final int producedLength) { + writeSequence(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + @Override + protected void applyArrayToOutput(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + writeArray(source, sourceOffset, sourceLength, output, outputOffset, producedLength); + } + + private void writeSequence(final CharSequence source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeForwardSequence(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + copySource(source, sourceOffset, sourceLength, output, outputOffset); + } + } + + private void writeArray(final char[] source, final int sourceOffset, final int sourceLength, + final char[] output, final int outputOffset, final int producedLength) { + if (!writeForwardArray(this.opcodes, this.operands, source, sourceOffset, sourceLength, output, + outputOffset, producedLength)) { + System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength); + } + } + } + + private static boolean writeForwardSequence(final char[] opcodes, final int[] operands, + final CharSequence source, final int sourceOffset, final int sourceLength, final char[] output, + final int outputOffset, final int producedLength) { + int currentLength = sourceLength; + int position = 0; + int sourceIndex = 0; + int outputIndex = 0; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceLength - sourceIndex); + copySource(source, sourceOffset + sourceIndex, skipCount, output, outputOffset + outputIndex); + sourceIndex += skipCount; + outputIndex += skipCount; + position = position + operand - 1; + break; + case DELETE_OPCODE: + if (position < 0 || position > currentLength) { + return false; + } + final int deletedLength = Math.min(operand, currentLength - position); + if (sourceIndex + deletedLength > sourceLength) { + return false; + } + sourceIndex += deletedLength; + currentLength -= deletedLength; + position--; + break; + case INSERT_OPCODE: + if (position < 0 || position > currentLength || outputIndex >= producedLength) { + return false; + } + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + currentLength++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceIndex >= sourceLength + || outputIndex >= producedLength) { + return false; + } + sourceIndex++; + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position++; + } + final int remainingLength = sourceLength - sourceIndex; + if (remainingLength > producedLength - outputIndex) { + return false; + } + copySource(source, sourceOffset + sourceIndex, remainingLength, output, outputOffset + outputIndex); + return outputIndex + remainingLength == producedLength; + } + + private static boolean writeForwardArray(final char[] opcodes, final int[] operands, final char[] source, + final int sourceOffset, final int sourceLength, final char[] output, final int outputOffset, + final int producedLength) { + int currentLength = sourceLength; + int position = 0; + int sourceIndex = 0; + int outputIndex = 0; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceLength - sourceIndex); + System.arraycopy(source, sourceOffset + sourceIndex, output, outputOffset + outputIndex, + skipCount); + sourceIndex += skipCount; + outputIndex += skipCount; + position = position + operand - 1; + break; + case DELETE_OPCODE: + if (position < 0 || position > currentLength) { + return false; + } + final int deletedLength = Math.min(operand, currentLength - position); + if (sourceIndex + deletedLength > sourceLength) { + return false; + } + sourceIndex += deletedLength; + currentLength -= deletedLength; + position--; + break; + case INSERT_OPCODE: + if (position < 0 || position > currentLength || outputIndex >= producedLength) { + return false; + } + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + currentLength++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceIndex >= sourceLength + || outputIndex >= producedLength) { + return false; + } + sourceIndex++; + output[outputOffset + outputIndex] = (char) operand; + outputIndex++; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position++; + } + final int remainingLength = sourceLength - sourceIndex; + if (remainingLength > producedLength - outputIndex) { + return false; + } + System.arraycopy(source, sourceOffset + sourceIndex, output, outputOffset + outputIndex, remainingLength); + return outputIndex + remainingLength == producedLength; + } + + private static boolean writeBackwardSequence(final char[] opcodes, final int[] operands, + final CharSequence source, final int sourceOffset, final int sourceLength, final char[] output, + final int outputOffset, final int producedLength) { + int currentLength = sourceLength; + int position = sourceLength - 1; + int sourceEnd = sourceLength; + int outputEnd = producedLength; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceEnd); + sourceEnd -= skipCount; + outputEnd -= skipCount; + if (outputEnd < 0) { + return false; + } + copySource(source, sourceOffset + sourceEnd, skipCount, output, outputOffset + outputEnd); + position = position - operand + 1; + break; + case DELETE_OPCODE: + final int deleteEndExclusive = position + 1; + position -= operand - 1; + if (position < 0 || position > currentLength || position > deleteEndExclusive) { + return false; + } + final int deletedLength = Math.min(deleteEndExclusive, currentLength) - position; + if (sourceEnd < deletedLength) { + return false; + } + sourceEnd -= deletedLength; + currentLength -= deletedLength; + break; + case INSERT_OPCODE: + if (position < -1 || position >= currentLength || outputEnd <= 0) { + return false; + } + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + currentLength++; + position++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceEnd <= 0 || outputEnd <= 0) { + return false; + } + sourceEnd--; + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position--; + } + if (sourceEnd != outputEnd) { + return false; + } + copySource(source, sourceOffset, sourceEnd, output, outputOffset); + return true; + } + + private static boolean writeBackwardArray(final char[] opcodes, final int[] operands, final char[] source, + final int sourceOffset, final int sourceLength, final char[] output, final int outputOffset, + final int producedLength) { + int currentLength = sourceLength; + int position = sourceLength - 1; + int sourceEnd = sourceLength; + int outputEnd = producedLength; + for (int index = 0; index < opcodes.length; index++) { + final char opcode = opcodes[index]; + final int operand = operands[index]; + switch (opcode) { + case SKIP_OPCODE: + final int skipCount = Math.min(operand, sourceEnd); + sourceEnd -= skipCount; + outputEnd -= skipCount; + if (outputEnd < 0) { + return false; + } + System.arraycopy(source, sourceOffset + sourceEnd, output, outputOffset + outputEnd, skipCount); + position = position - operand + 1; + break; + case DELETE_OPCODE: + final int deleteEndExclusive = position + 1; + position -= operand - 1; + if (position < 0 || position > currentLength || position > deleteEndExclusive) { + return false; + } + final int deletedLength = Math.min(deleteEndExclusive, currentLength) - position; + if (sourceEnd < deletedLength) { + return false; + } + sourceEnd -= deletedLength; + currentLength -= deletedLength; + break; + case INSERT_OPCODE: + if (position < -1 || position >= currentLength || outputEnd <= 0) { + return false; + } + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + currentLength++; + position++; + break; + case REPLACE_OPCODE: + if (position < 0 || position >= currentLength || sourceEnd <= 0 || outputEnd <= 0) { + return false; + } + sourceEnd--; + outputEnd--; + output[outputOffset + outputEnd] = (char) operand; + break; + default: + throw new AssertionError(MSG_OPCODE + opcode); + } + position--; + } + if (sourceEnd != outputEnd) { + return false; + } + System.arraycopy(source, sourceOffset, output, outputOffset, sourceEnd); + return true; + } +} diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrie.java b/src/main/java/org/egothor/stemmer/FrequencyTrie.java index 9c1cae4..9ff0e11 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrie.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrie.java @@ -89,7 +89,7 @@ import org.egothor.stemmer.trie.ReductionSignature; * * @param+ * This method bypasses {@link TrieMetadata#caseProcessingMode()} and + * {@link TrieMetadata#diacriticProcessingMode()}. Callers must supply input + * normalized exactly as required by this trie's metadata. It is intended for + * hot paths where normalization is guaranteed by an upstream tokenizer or + * benchmark corpus and repeated lookup-time normalization would be redundant. + *
+ * + * @param key already-normalized key to resolve + * @return most frequent value, or {@code null} if the key does not exist or no + * value is stored at the addressed node + * @throws NullPointerException if {@code key} is {@code null} + */ + public V getNormalized(final CharSequence key) { + Objects.requireNonNull(key, ARG_KEY); + final CompiledNode+ * This overload keeps high-volume string lookup on a monomorphic path and + * avoids the {@link CharSequence} dispatch used by the general overload. + * Callers must supply input normalized exactly as required by this trie's + * metadata. + *
+ * + * @param key already-normalized key to resolve + * @return most frequent value, or {@code null} if the key does not exist or no + * value is stored at the addressed node + * @throws NullPointerException if {@code key} is {@code null} + */ + public V getNormalizedString(final String key) { + Objects.requireNonNull(key, ARG_KEY); + final CompiledNode+ * The method preserves logical keys, local value counts, trie metadata, and the + * supplied reduction settings. It is intended for runtime specialization, such + * as replacing serialized patch-command strings with precompiled patch command + * objects without changing the persisted binary trie format. + *
+ * + * @param source source compiled trie + * @param arrayFactory array factory for mapped values + * @param reductionSettings reduction settings for the mapped trie + * @param valueMapper value mapping function + * @param- * This is the branch-free instance-level fast path for repeated patch - * application in a known traversal direction. + * This is the instance-level fast path for repeated patch application in a + * known traversal direction. It avoids the static API null and direction + * validation path and calls the selected decoder directly. *
* * @param source original source word * @param patchCommand compact patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should compile + * {@code patchCommand} once through {@link #compile(String)} and + * reuse {@link CompiledPatchCommand#apply(String)}. The + * String-based application path reparses the patch command on every + * call and is kept only for source compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public String applyWithConfiguredDirection(final String source, final String patchCommand) { if (source == null) { return null; } - return this.applyStrategy.apply(source, patchCommand); + if (this.backwardTraversal) { + return applyBackwardNonNull(source, patchCommand); + } + return applyForwardNonNull(source, patchCommand); + } + + /** + * Compiles a patch command for repeated application with this encoder + * instance traversal direction. + * + * @param patchCommand compact patch command + * @return immutable compiled patch command + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public CompiledPatchCommand compile(final String patchCommand) { + return CompiledPatchCommand.compile(patchCommand, this.traversalDirection); } /** @@ -326,7 +325,14 @@ public final class PatchCommandEncoder { * @param source original source word * @param patchCommand compact patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should use + * {@link CompiledPatchCommand#compile(String, WordTraversalDirection)} + * once and then reuse {@link CompiledPatchCommand#apply(String)}. + * This method repeatedly interprets the serialized patch-command + * string and is retained only for compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static String apply(final String source, final String patchCommand) { return apply(source, patchCommand, WordTraversalDirection.BACKWARD); } @@ -343,14 +349,41 @@ public final class PatchCommandEncoder { * @param patchCommand compact patch command * @param traversalDirection traversal direction used by the patch command * @return transformed word, or {@code null} when {@code source} is {@code null} + * @deprecated Since 2.3.0. Runtime stemming should use + * {@link CompiledPatchCommand#compile(String, WordTraversalDirection)} + * once and then reuse {@link CompiledPatchCommand#apply(String)}. + * This method repeatedly interprets the serialized patch-command + * string and is retained only for compatibility before the 3.0.0 + * migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static String apply(final String source, final String patchCommand, final WordTraversalDirection traversalDirection) { Objects.requireNonNull(traversalDirection, "traversalDirection"); if (source == null) { return null; } - return applyStrategyFor(traversalDirection).apply(source, patchCommand); + if (traversalDirection == WordTraversalDirection.BACKWARD) { + return applyBackwardNonNull(source, patchCommand); + } + return applyForwardNonNull(source, patchCommand); + } + + /** + * Compiles a patch command for repeated application with the supplied + * traversal direction. + * + * @param patchCommand compact patch command + * @param traversalDirection traversal direction used by the patch command + * @return immutable compiled patch command + * @throws NullPointerException if {@code traversalDirection} is + * {@code null} + * @throws IllegalArgumentException if the serialized command contains an + * unsupported opcode or invalid NOOP argument + */ + public static CompiledPatchCommand compile(final String patchCommand, + final WordTraversalDirection traversalDirection) { + return CompiledPatchCommand.compile(patchCommand, traversalDirection); } /** @@ -371,7 +404,15 @@ public final class PatchCommandEncoder { * @param outputLength writable output capacity * @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY} * when {@code outputLength} is too small + * @deprecated Since 2.3.0. Compile {@code patchCommand} once through + * {@link #compile(String, WordTraversalDirection)} and call + * {@link CompiledPatchCommand#applyTo(CharSequence, char[], int, int)} + * or + * {@link CompiledPatchCommand#applyTo(CharSequence, int, int, char[], int, int)}. + * This String-based method reparses patch commands on every call and + * is kept only for compatibility before the 3.0.0 migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static int applyTo(final CharSequence source, final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset, final int outputLength) { @@ -405,7 +446,13 @@ public final class PatchCommandEncoder { * when {@code outputLength} is too small * @throws IllegalArgumentException when source and output ranges overlap in the * same array + * @deprecated Since 2.3.0. Compile {@code patchCommand} once through + * {@link #compile(String, WordTraversalDirection)} and call + * {@link CompiledPatchCommand#applyTo(char[], int, int, char[], int, int)}. + * This String-based method reparses patch commands on every call and + * is kept only for compatibility before the 3.0.0 migration. */ + @Deprecated(since = "2.3.0", forRemoval = false) public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength, final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset, final int outputLength) { @@ -484,36 +531,33 @@ public final class PatchCommandEncoder { /** * Applies a patch command using the historical backward Egothor semantics. * - * @param source original source word in legacy backward logical space + * @param source non-null original source word in legacy backward logical + * space * @param patchCommand compact patch command - * @return transformed word, or {@code null} when {@code source} is {@code null} + * @return transformed word */ - private static String applyBackward(final String source, final String patchCommand) { - if (source == null) { - return null; - } - if (patchCommand == null || patchCommand.isEmpty()) { + private static String applyBackwardNonNull(final String source, final String patchCommand) { + if (patchCommand == null) { return source; } - if (NOOP_PATCH.equals(patchCommand)) { + final int patchLength = patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { return source; } - if ((patchCommand.length() & 1) != 0) { - return source; - } - if (patchCommand.length() == 2) { + if (patchLength == 2) { return applySingleBackwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1)); } - final StringBuilder result = new StringBuilder(source); - if (result.isEmpty()) { - return applyBackwardToEmptySource(result, patchCommand); + if (source.isEmpty()) { + return applyBackwardToEmptySource(patchCommand); } + final StringBuilder result = new StringBuilder(source); + int position = result.length() - 1; try { - for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); final char argument = patchCommand.charAt(patchIndex + 1); @@ -547,12 +591,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return source; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } position--; @@ -567,36 +611,32 @@ public final class PatchCommandEncoder { /** * Applies a patch command using forward traversal semantics. * - * @param source original source word + * @param source non-null original source word * @param patchCommand compact patch command - * @return transformed word, or {@code null} when {@code source} is {@code null} + * @return transformed word */ - private static String applyForward(final String source, final String patchCommand) { - if (source == null) { - return null; - } - if (patchCommand == null || patchCommand.isEmpty()) { + private static String applyForwardNonNull(final String source, final String patchCommand) { + if (patchCommand == null) { return source; } - if (NOOP_PATCH.equals(patchCommand)) { + final int patchLength = patchCommand.length(); + if (patchLength == 0 || (patchLength & 1) != 0) { return source; } - if ((patchCommand.length() & 1) != 0) { - return source; - } - if (patchCommand.length() == 2) { + if (patchLength == 2) { return applySingleForwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1)); } - final StringBuilder result = new StringBuilder(source); - if (result.isEmpty()) { - return applyForwardToEmptySource(result, patchCommand); + if (source.isEmpty()) { + return applyForwardToEmptySource(patchCommand); } + final StringBuilder result = new StringBuilder(source); + int position = 0; try { - for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { + for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); final char argument = patchCommand.charAt(patchIndex + 1); @@ -628,12 +668,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return source; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } position++; @@ -751,12 +791,12 @@ public final class PatchCommandEncoder { * behavior for index-invalid commands. * * - * @param result empty result builder * @param patchCommand compact patch command * @return transformed word, or the original empty word when the patch is * malformed */ - private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) { + private static String applyBackwardToEmptySource(final String patchCommand) { + final StringBuilder result = new StringBuilder(patchCommand.length() >> 1); try { for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); @@ -774,12 +814,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return ""; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } } } catch (IndexOutOfBoundsException exception) { @@ -792,12 +832,12 @@ public final class PatchCommandEncoder { /** * Applies a forward patch command to an empty source word. * - * @param result empty result builder * @param patchCommand compact patch command * @return transformed word, or the original empty word when the patch is * malformed */ - private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) { + private static String applyForwardToEmptySource(final String patchCommand) { + final StringBuilder result = new StringBuilder(patchCommand.length() >> 1); try { for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { final char opcode = patchCommand.charAt(patchIndex); @@ -815,12 +855,12 @@ public final class PatchCommandEncoder { case NOOP_OPCODE: if (argument != NOOP_ARGUMENT) { - throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument); + throw new IllegalArgumentException(MSG_NOOP + argument); } return ""; default: - throw new IllegalArgumentException("Unsupported patch opcode: " + opcode); + throw new IllegalArgumentException(MSG_OPCODE + opcode); } } } catch (IndexOutOfBoundsException exception) { @@ -1535,16 +1575,6 @@ public final class PatchCommandEncoder { } } - /** - * Returns the direction-specialized apply strategy. - * - * @param traversalDirection requested traversal direction - * @return branch-free apply strategy for that direction - */ - private static ApplyStrategy applyStrategyFor(final WordTraversalDirection traversalDirection) { - return traversalDirection == WordTraversalDirection.BACKWARD ? BACKWARD_APPLY_STRATEGY : FORWARD_APPLY_STRATEGY; - } - /** * Decodes a compact count argument used by skip and delete instructions. * diff --git a/src/main/java/org/egothor/stemmer/ReductionSettings.java b/src/main/java/org/egothor/stemmer/ReductionSettings.java index d6e47dd..8d8b1cb 100644 --- a/src/main/java/org/egothor/stemmer/ReductionSettings.java +++ b/src/main/java/org/egothor/stemmer/ReductionSettings.java @@ -42,10 +42,13 @@ import java.util.Objects; * @param reductionMode reduction mode * @param dominantWinnerMinPercent minimum dominant winner percentage * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio + * @param contractUniformSubtrees whether compilation may contract a subtree + * whose reachable terminal values all contain + * the same single value */ @SuppressWarnings("PMD.LongVariable") public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerMinPercent, - int dominantWinnerOverSecondRatio) { + int dominantWinnerOverSecondRatio, boolean contractUniformSubtrees) { /** * Default minimum dominant winner percentage. @@ -65,12 +68,14 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM * the inclusive range {@code 1..100} * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio, must * be at least {@code 1} + * @param contractUniformSubtrees whether uniform subtrees may be + * contracted into accepting leaves * @throws NullPointerException if {@code reductionMode} is {@code null} * @throws IllegalArgumentException if any numeric value is outside the valid * range */ public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent, - final int dominantWinnerOverSecondRatio) { + final int dominantWinnerOverSecondRatio, final boolean contractUniformSubtrees) { this.reductionMode = Objects.requireNonNull(reductionMode, "reductionMode"); if (dominantWinnerMinPercent < 1 || dominantWinnerMinPercent > 100) { throw new IllegalArgumentException("dominantWinnerMinPercent must be in range 1..100."); @@ -80,6 +85,19 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM } this.dominantWinnerMinPercent = dominantWinnerMinPercent; this.dominantWinnerOverSecondRatio = dominantWinnerOverSecondRatio; + this.contractUniformSubtrees = contractUniformSubtrees; + } + + /** + * Creates a new instance without uniform-subtree contraction. + * + * @param reductionMode reduction mode + * @param dominantWinnerMinPercent minimum dominant winner percentage + * @param dominantWinnerOverSecondRatio minimum winner-over-second ratio + */ + public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent, + final int dominantWinnerOverSecondRatio) { + this(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio, false); } /** @@ -93,4 +111,23 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM return new ReductionSettings(reductionMode, DEFAULT_DOMINANT_WINNER_MIN_PERCENT, DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO); } + + /** + * Returns settings that run uniform-subtree contraction before the configured + * subtree-merging mode. + * + *+ * This is intended for Radixor patch-command tries, where a contracted accepting + * leaf can safely represent a subtree whose reachable entries all use the same + * patch command. + *
+ * + * @param settings base settings + * @return equivalent settings with uniform-subtree contraction enabled + */ + /* default */ static ReductionSettings withUniformSubtreeContraction(final ReductionSettings settings) { + Objects.requireNonNull(settings, "settings"); + return new ReductionSettings(settings.reductionMode(), settings.dominantWinnerMinPercent(), + settings.dominantWinnerOverSecondRatio(), true); + } } diff --git a/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java b/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java index 8d7e0cb..3da92b8 100644 --- a/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java +++ b/src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java @@ -349,6 +349,7 @@ public final class StemmerKnowledgeExperiment { * @param trie compiled trie under test * @return immutable counts for this single input */ + @SuppressWarnings("deprecation") private static EvaluationCounts evaluateInput(final String input, final String expectedStem, final FrequencyTrie+ * The text dictionary is still compiled through the canonical serialized + * patch-command representation. The returned trie replaces each stored + * serialized patch command with a {@link CompiledPatchCommand} so repeated + * runtime stemming does not parse patch-command strings. + *
+ * + * @param language bundled language dictionary + * @param storeOriginal whether the stem itself should be inserted using the + * canonical no-op patch command + * @param reductionSettings reduction settings + * @return compiled patch-command trie with runtime-specialized values + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if the dictionary cannot be found or read + */ + public static FrequencyTrie+ * Equal textual patch commands are compiled once and shared by all trie values + * that reference them. The returned trie preserves the source trie keys, + * metadata, traversal direction, counts, and reduction settings. + *
+ * + * @param trie source trie containing textual patch commands + * @return equivalent trie containing compiled patch commands + * @throws NullPointerException if {@code trie} is {@code null} + */ + private static FrequencyTrie