Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: prepare Radixor 3.0.0 with contracted tries and compiled patch commands

Browse Source 
Introduce contracted compiled patch tries for faster lookup, make compiled
patch commands the primary runtime path, refresh stemmer benchmarks and
documentation, and restructure the documentation for 3.0.0 onboarding.

BREAKING CHANGE: Radixor 3.0.0 promotes compiled patch-command APIs and
new compiled trie artifacts as the primary runtime integration model.
This commit is contained in:
Leo Galambos
2026-07-03 18:44:39 +02:00
parent df4552b113
commit 38620d7e71
101 changed files with 11235 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java
View File

@@ -113,12 +113,12 @@ final class BenchmarkCorpusSupport {
dictionaryBuilder.append(stem);
lookupKeys.add(stem);
for (String variant : variants) {
dictionaryBuilder.append(' ').append(variant);
dictionaryBuilder.append('\t').append(variant);
lookupKeys.add(variant);
}
final String homograph = createHomograph(index);
dictionaryBuilder.append(' ').append(homograph);
dictionaryBuilder.append('\t').append(homograph);
lookupKeys.add(homograph);
ambiguousLookupKeys.add(homograph);

185
src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java Normal file
View File

@@ -0,0 +1,185 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.util.Objects;
/**
* Reusable deterministic token sequence for benchmark-only token streams.
*
* <p>
* The sequence keeps stable token ordering and offset progression while avoiding
* per-token object creation during iteration.
* </p>
*/
final class BenchmarkTokenSequence {
/**
* Shared backing corpus as character arrays.
*/
private char[][] tokenCharacters;
/**
* Number of active tokens in the sequence.
*/
private int tokenCount;
/**
* Cursor index for the currently emitted token.
*/
private int cursor;
/**
* Current token character array.
*/
private char[] currentToken;
/**
* Start offset of the current token.
*/
private int currentStartOffset;
/**
* End offset of the current token.
*/
private int currentEndOffset;
/**
* Offset of the next token start.
*/
private int nextOffset;
/**
* Creates a reusable token sequence.
*
* @param tokens token corpus source
*/
BenchmarkTokenSequence(final String[] tokens) {
setTokens(tokens);
}
/**
* Sets a new token corpus for this sequence.
*
* <p>
* The sequence stores copied character arrays so token reads can be reused
* without creating per-token objects during benchmark iteration.
* </p>
*
* @param tokens new token corpus
*/
void setTokens(final String[] tokens) {
Objects.requireNonNull(tokens, "tokens");
this.tokenCharacters = new char[tokens.length][];
for (int index = 0; index < tokens.length; index++) {
final String token = Objects.requireNonNull(tokens[index], "tokens[" + index + "]");
this.tokenCharacters[index] = token.toCharArray();
}
this.tokenCount = this.tokenCharacters.length;
reset();
}
/**
* Resets stream position for reuse.
*/
void reset() {
this.cursor = 0;
this.nextOffset = 0;
this.currentStartOffset = 0;
this.currentEndOffset = 0;
this.currentToken = null;
}
/**
* Returns whether at least one token remains in the sequence.
*
* @return true if a token can be emitted
*/
boolean hasNext() {
return this.cursor < this.tokenCount;
}
/**
* Advances to the next token.
*
* @return true if a token was emitted
*/
boolean advance() {
if (!hasNext()) {
return false;
}
final char[] token = this.tokenCharacters[this.cursor];
this.currentToken = token;
this.currentStartOffset = this.nextOffset;
this.currentEndOffset = this.currentStartOffset + token.length;
this.nextOffset = this.currentEndOffset + 1;
this.cursor++;
return true;
}
/**
* Returns the current token in the sequence.
*
* @return current token character array
*/
char[] currentToken() {
return this.currentToken;
}
/**
* Returns current token start offset for token stream attributes.
*
* @return start offset
*/
int currentStartOffset() {
return this.currentStartOffset;
}
/**
* Returns current token end offset for token stream attributes.
*
* @return end offset
*/
int currentEndOffset() {
return this.currentEndOffset;
}
/**
* Returns final stream offset value used by {@code end()}.
*
* @return final offset
*/
int endOffset() {
return this.nextOffset > 0 ? this.nextOffset - 1 : 0;
}
}

146
src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenStream.java Normal file
View File

@@ -0,0 +1,146 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
/**
* Reusable Lucene {@link TokenStream} backed by a deterministic token array.
*
* <p>
* Instances are mutable and intended for one JMH worker thread. The stream
* copies configured token text into reusable character storage so benchmark
* iteration can replay the same token sequence without mutating the shared
* source array.
* </p>
*/
final class BenchmarkTokenStream extends TokenStream {
/**
* Current token text attribute.
*/
private final CharTermAttribute charTermAttribute;
/**
* Offset attribute used by Lucene filters that inspect offsets.
*/
private final OffsetAttribute offsetAttribute;
/**
* Position increment attribute for one-token-at-a-time streams.
*/
private final PositionIncrementAttribute positionIncrementAttribute;
/**
* Reusable token sequence.
*/
private final BenchmarkTokenSequence tokenSequence;
/**
* Creates a stream over the supplied tokens.
*
* @param tokens initial token corpus
*/
BenchmarkTokenStream(final String[] tokens) {
this.tokenSequence = new BenchmarkTokenSequence(tokens);
this.charTermAttribute = addAttribute(CharTermAttribute.class);
this.offsetAttribute = addAttribute(OffsetAttribute.class);
this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
}
/**
* Replaces the configured token corpus.
*
* @param tokens new token corpus
*/
void setTokens(final String[] tokens) {
this.tokenSequence.setTokens(tokens);
}
/**
* Returns whether all configured tokens have been emitted.
*
* @return {@code true} after the current pass is exhausted
*/
boolean isDrained() {
return !this.tokenSequence.hasNext();
}
/**
* {@inheritDoc}
*/
@Override
public boolean incrementToken() throws IOException {
if (!this.tokenSequence.advance()) {
return false;
}
clearAttributes();
final char[] token = this.tokenSequence.currentToken();
this.charTermAttribute.copyBuffer(token, 0, token.length);
this.positionIncrementAttribute.setPositionIncrement(1);
this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset());
return true;
}
/**
* {@inheritDoc}
*/
@Override
public void reset() throws IOException {
super.reset();
this.tokenSequence.reset();
}
/**
* {@inheritDoc}
*/
@Override
public void end() throws IOException {
super.end();
final int endOffset = this.tokenSequence.endOffset();
this.offsetAttribute.setOffset(endOffset, endOffset);
}
/**
* {@inheritDoc}
*/
@Override
public void close() throws IOException {
super.close();
this.charTermAttribute.setEmpty();
}
}

98
src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java
View File

@@ -30,43 +30,24 @@
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.io.IOException;
import org.egothor.stemmer.StemmerPatchTrieLoader;
/**
* Builds a deterministic English token corpus for side-by-side stemming
* benchmarks.
* benchmarks from the bundled Radixor English dictionary resource.
*
* <p>
* The generated corpus mixes:
* </p>
* <ul>
* <li>simple inflections</li>
* <li>common derivational forms</li>
* <li>US/UK spelling families</li>
* <li>forms that are suitable for comparison against the bundled
* {@code US_UK_PROFI} Radixor dictionary</li>
* </ul>
*
* <p>
* The goal is not to simulate natural language frequency distribution exactly,
* but to provide a stable and reproducible comparison workload for benchmark
* runs and regression tracking.
* The dictionary resource stores the expected stem as the first tab-separated
* field on each line and its surface variants on the same line. This helper
* uses only token/root pairs where the token differs from the expected root for
* timing. Resources smaller than the shared timing minimum are repeated
* deterministically by {@link LanguageBenchmarkCorpus}.
* </p>
*/
final class EnglishComparisonCorpus {
/**
* Canonical lexical bases used to generate the token workload.
*/
private static final String[] BASES = { "analyze", "analyse", "color", "colour", "center", "centre", "organize",
"organise", "optimize", "optimise", "characterize", "characterise", "connect", "construct", "compute",
"design", "develop", "engineer", "govern", "improve", "index", "inform", "manage", "model", "observe",
"operate", "perform", "predict", "prepare", "process", "project", "protect", "publish", "query", "reduce",
"refresh", "render", "resolve", "return", "search", "select", "signal", "store", "structure", "support",
"transform", "update", "validate", "value" };
/**
* Utility class.
*/
@@ -77,64 +58,21 @@ final class EnglishComparisonCorpus {
/**
* Creates a deterministic token corpus for English stemming comparison.
*
* @param familyCount number of generated lexical families
* @return token array in stable order
* @throws IOException if the bundled English resource cannot be read
*/
static String[] createTokens(final int familyCount) {
if (familyCount < 1) {
throw new IllegalArgumentException("familyCount must be at least 1.");
}
final List<String> tokens = new ArrayList<>(familyCount * 14);
for (int index = 0; index < familyCount; index++) {
final String base = createBase(index);
tokens.add(base);
tokens.add(base + "s");
tokens.add(base + "ed");
tokens.add(base + "ing");
tokens.add(base + "er");
tokens.add(base + "ers");
tokens.add(base + "ly");
tokens.add(base + "ness");
tokens.add(base + "ment");
tokens.add(base + "ments");
tokens.add(base + "able");
tokens.add(base + "ability");
if (base.endsWith("ize")) {
tokens.add(base.substring(0, base.length() - 3) + "isation");
tokens.add(base.substring(0, base.length() - 3) + "ised");
}
if (base.endsWith("ise")) {
tokens.add(base.substring(0, base.length() - 3) + "ization");
tokens.add(base.substring(0, base.length() - 3) + "ized");
}
}
return tokens.toArray(String[]::new);
static String[] createTokens() throws IOException {
return createCorpus().tokens();
}
/**
* Creates one deterministic base token.
* Creates a deterministic changed-token corpus and expected root array for
* English stemming comparison.
*
* @param index base ordinal
* @return generated lexical base
* @return token corpus with expected roots
* @throws IOException if the bundled English resource cannot be read
*/
private static String createBase(final int index) {
return (BASES[index % BASES.length] + suffix(index)).toLowerCase(Locale.ROOT);
}
/**
* Creates a compact discriminator suffix so that large corpora remain unique
* while retaining stable lexical families.
*
* @param value ordinal value
* @return compact discriminator
*/
private static String suffix(final int value) {
return Integer.toString(value, Character.MAX_RADIX);
static LanguageBenchmarkCorpus.Corpus createCorpus() throws IOException {
return LanguageBenchmarkCorpus.createChangedCorpus(StemmerPatchTrieLoader.Language.US_UK);
}
}

431
src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java Normal file
View File

@@ -0,0 +1,431 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import org.egothor.stemmer.CompiledPatchCommand;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.FrequencyTrieBuilders;
import org.egothor.stemmer.PatchCommandEncoder;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.ReductionSettings;
import org.egothor.stemmer.StemmerDictionaryParser;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.egothor.stemmer.WordTraversalDirection;
import org.openjdk.jmh.annotations.AuxCounters;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
/**
* Measures Radixor English stemming quality and changed-token speed when the
* runtime trie is trained from a deterministic percentage of dictionary rows.
*
* <p>
* The measured stemmer always uses {@link CompiledPatchCommand} values. Quality
* is evaluated against the complete English dictionary corpus, while speed is
* measured over the complete changed-token English corpus used by the comparison
* benchmarks.
* </p>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
public class EnglishRadixorDictionaryCoverageBenchmark {
/**
* Shared benchmark state for one dictionary-row coverage percentage.
*/
@State(Scope.Benchmark)
public static class CoverageState {
/**
* Percentage of parsed English dictionary rows used to build the Radixor trie.
*/
@Param({ "100", "90", "80", "70", "60", "50", "40", "30", "20", "10" })
public int coveragePercent;
/**
* Full English corpus used for exact-root accounting.
*/
private LanguageBenchmarkCorpus.Corpus fullCorpus;
/**
* Complete changed-token English corpus used for speed measurement.
*/
private LanguageBenchmarkCorpus.Corpus changedCorpus;
/**
* Radixor stemmer backed by a trie built from selected dictionary rows.
*/
private RadixorBenchmarkStemmer stemmer;
/**
* Parsed dictionary row count before deterministic coverage selection.
*/
private int totalRowCount;
/**
* Selected dictionary row count for the configured coverage percentage.
*/
private int selectedRowCount;
/**
* Builds the reduced dictionary trie and shared corpora before measurement.
*
* @throws IOException if the English dictionary resource cannot be read
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
final List<DictionaryRow> rows = readEnglishRows();
this.totalRowCount = rows.size();
final List<DictionaryRow> selectedRows = selectRows(rows, this.coveragePercent);
this.selectedRowCount = selectedRows.size();
this.fullCorpus = LanguageBenchmarkCorpus.createFullCorpus(StemmerPatchTrieLoader.Language.US_UK);
this.changedCorpus = LanguageBenchmarkCorpus.createChangedCorpus(StemmerPatchTrieLoader.Language.US_UK);
this.stemmer = new RadixorBenchmarkStemmer(buildCompiledTrie(selectedRows));
}
}
/**
* JMH auxiliary counters for dictionary-row coverage and exact-root agreement.
*/
@State(Scope.Thread)
@AuxCounters(AuxCounters.Type.EVENTS)
public static class CoverageCounters {
/**
* Number of exact output/root matches over the full dictionary corpus.
*/
public long correctMatches;
/**
* Number of evaluated tokens over the full dictionary corpus.
*/
public long evaluatedTokens;
/**
* Number of exact output/root matches where token and root differ.
*/
public long changedCorrectMatches;
/**
* Number of evaluated tokens where token and root differ.
*/
public long changedEvaluatedTokens;
/**
* Number of exact output/root matches where token already equals root.
*/
public long rootPreservedMatches;
/**
* Number of evaluated tokens where token already equals root.
*/
public long rootEvaluatedTokens;
/**
* Number of parsed dictionary rows used for trie construction.
*/
public long selectedRows;
/**
* Total number of parsed dictionary rows available.
*/
public long totalRows;
/**
* Resets counters before each measured iteration.
*/
@Setup(Level.Iteration)
public void reset() {
this.correctMatches = 0L;
this.evaluatedTokens = 0L;
this.changedCorrectMatches = 0L;
this.changedEvaluatedTokens = 0L;
this.rootPreservedMatches = 0L;
this.rootEvaluatedTokens = 0L;
this.selectedRows = 0L;
this.totalRows = 0L;
}
}
/**
* Measures direct Radixor stemming over the complete English changed-token
* corpus.
*
* @param state shared coverage state
* @param blackhole result sink
*/
@Benchmark
public void changedTokenStemmingSpeed(final CoverageState state, final Blackhole blackhole) {
final String[] tokens = state.changedCorpus.tokens();
final RadixorBenchmarkStemmer stemmer = state.stemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Measures exact-root agreement over the complete English dictionary corpus.
*
* @param state shared coverage state
* @param counters auxiliary exact-root counters
* @param blackhole result sink
* @return exact-root match count for one benchmark operation
*/
@Benchmark
public int exactRootAgreement(final CoverageState state, final CoverageCounters counters,
final Blackhole blackhole) {
final QualityCounts counts = evaluate(state.fullCorpus, state.stemmer, blackhole);
counters.correctMatches += counts.correctMatches();
counters.evaluatedTokens += counts.evaluatedTokens();
counters.changedCorrectMatches += counts.changedCorrectMatches();
counters.changedEvaluatedTokens += counts.changedEvaluatedTokens();
counters.rootPreservedMatches += counts.rootPreservedMatches();
counters.rootEvaluatedTokens += counts.rootEvaluatedTokens();
counters.selectedRows += state.selectedRowCount;
counters.totalRows += state.totalRowCount;
return counts.correctMatches();
}
private static QualityCounts evaluate(final LanguageBenchmarkCorpus.Corpus corpus,
final RadixorBenchmarkStemmer stemmer, final Blackhole blackhole) {
final String[] tokens = corpus.tokens();
final String[] roots = corpus.expectedRoots();
int correct = 0;
int changedCorrect = 0;
int changedEvaluated = 0;
int rootPreserved = 0;
int rootEvaluated = 0;
for (int index = 0; index < tokens.length; index++) {
final String token = tokens[index];
final String root = roots[index];
final String actual = stemmer.stem(token);
blackhole.consume(actual);
final boolean exact = Objects.equals(root, actual);
if (exact) {
correct++;
}
if (Objects.equals(token, root)) {
rootEvaluated++;
if (exact) {
rootPreserved++;
}
} else {
changedEvaluated++;
if (exact) {
changedCorrect++;
}
}
}
return new QualityCounts(correct, tokens.length, changedCorrect, changedEvaluated, rootPreserved,
rootEvaluated);
}
private static FrequencyTrie<CompiledPatchCommand> buildCompiledTrie(final List<DictionaryRow> rows) {
final ReductionSettings settings = new ReductionSettings(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS,
ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT,
ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO,
true);
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, settings,
WordTraversalDirection.BACKWARD);
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
.traversalDirection(WordTraversalDirection.BACKWARD)
.build();
for (DictionaryRow row : rows) {
builder.put(row.stem(), encoder.encode(row.stem(), row.stem()));
for (String variant : row.variants()) {
if (!variant.equals(row.stem())) {
builder.put(variant, encoder.encode(variant, row.stem()));
}
}
}
final FrequencyTrie<String> trie = builder.build();
final Map<String, CompiledPatchCommand> compiledCommands = new HashMap<String, CompiledPatchCommand>(4096);
return FrequencyTrieBuilders.mapValues(trie, CompiledPatchCommand[]::new, trie.metadata().reductionSettings(),
patch -> compiledCommands.computeIfAbsent(patch,
value -> CompiledPatchCommand.compile(value, trie.traversalDirection())));
}
private static List<DictionaryRow> selectRows(final List<DictionaryRow> rows, final int coveragePercent) {
if (coveragePercent < 1 || coveragePercent > 100) {
throw new IllegalArgumentException("coveragePercent must be between 1 and 100.");
}
if (coveragePercent == 100) {
return List.copyOf(rows);
}
final int selectedCount = Math.max(1, Math.round(rows.size() * coveragePercent / 100.0F));
final List<DictionaryRow> rankedRows = new ArrayList<DictionaryRow>(rows);
rankedRows.sort(Comparator.comparingLong(DictionaryRow::rank).thenComparingInt(DictionaryRow::lineNumber));
final Set<Integer> selectedLineNumbers = new HashSet<Integer>(selectedCount);
for (int index = 0; index < selectedCount; index++) {
selectedLineNumbers.add(rankedRows.get(index).lineNumber());
}
final List<DictionaryRow> selectedRows = new ArrayList<DictionaryRow>(selectedCount);
for (DictionaryRow row : rows) {
if (selectedLineNumbers.contains(row.lineNumber())) {
selectedRows.add(row);
}
}
return selectedRows;
}
private static List<DictionaryRow> readEnglishRows() throws IOException {
final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath();
final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath);
if (resource == null) {
throw new IllegalStateException("Missing bundled English dictionary resource " + resourcePath + ".");
}
final List<DictionaryRow> rows = new ArrayList<DictionaryRow>(400_000);
try (InputStream inputStream = resource;
GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
InputStreamReader inputStreamReader = new InputStreamReader(gzipInputStream, StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(inputStreamReader)) {
StemmerDictionaryParser.parse(reader, resourcePath, (stem, variants, lineNumber) -> {
rows.add(new DictionaryRow(lineNumber, stem, variants, rank(lineNumber, stem, variants)));
});
}
return rows;
}
private static long rank(final int lineNumber, final String stem, final String[] variants) {
long hash = 0xcbf29ce484222325L;
hash = mix(hash, lineNumber);
hash = mix(hash, stem);
for (String variant : variants) {
hash = mix(hash, variant);
}
return hash;
}
private static long mix(final long hash, final int value) {
long result = hash;
result ^= value & 0xFFL;
result *= 0x100000001b3L;
result ^= value >>> 8 & 0xFFL;
result *= 0x100000001b3L;
result ^= value >>> 16 & 0xFFL;
result *= 0x100000001b3L;
result ^= value >>> 24 & 0xFFL;
result *= 0x100000001b3L;
return result;
}
private static long mix(final long hash, final String value) {
long result = hash;
for (int index = 0; index < value.length(); index++) {
final char character = value.charAt(index);
result ^= character & 0xFFL;
result *= 0x100000001b3L;
result ^= character >>> 8;
result *= 0x100000001b3L;
}
result ^= 0xFFL;
result *= 0x100000001b3L;
return result;
}
/**
* One parsed dictionary row with deterministic selection rank.
*
* @param lineNumber source dictionary line number
* @param stem canonical stem from the first column
* @param variants normalized variants from following columns
* @param rank deterministic selection rank
*/
private record DictionaryRow(int lineNumber, String stem, String[] variants, long rank) {
/**
* Creates one immutable dictionary row snapshot.
*
* @param lineNumber source dictionary line number
* @param stem canonical stem from the first column
* @param variants normalized variants from following columns
* @param rank deterministic selection rank
*/
DictionaryRow {
Objects.requireNonNull(stem, "stem");
variants = variants.clone();
}
@Override
public String[] variants() {
return this.variants.clone();
}
}
/**
* Exact-root accounting result for one quality operation.
*
* @param correctMatches exact-root matches for all tokens
* @param evaluatedTokens evaluated token count
* @param changedCorrectMatches exact-root matches for changed tokens
* @param changedEvaluatedTokens evaluated changed-token count
* @param rootPreservedMatches exact-root matches for root-equal tokens
* @param rootEvaluatedTokens evaluated root-equal token count
*/
private record QualityCounts(int correctMatches, int evaluatedTokens, int changedCorrectMatches,
int changedEvaluatedTokens, int rootPreservedMatches, int rootEvaluatedTokens) {
}
}

466
src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java
View File

@@ -32,140 +32,314 @@ package org.egothor.stemmer.benchmark;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishMinimalStemFilter;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.KStemFilter;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.egothor.stemmer.benchmark.snowball.ext.englishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.PatchCommandEncoder;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import org.tartarus.snowball.ext.englishStemmer;
import org.tartarus.snowball.ext.porterStemmer;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.StemmerDictionaryParser;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.StemmerPatchTrieLoader;
/**
* Compares English stemming throughput across Radixor and Snowball stemmers.
* Compares English stemming throughput across Radixor and selected Java
* algorithm paths with a shared deterministic corpus.
*
* <p>
* The benchmark processes the same deterministic token array with:
* The comparison uses one shared changed-token dictionary array for all methods:
* </p>
* <ul>
* <li>Radixor using bundled {@link StemmerPatchTrieLoader.Language#US_UK}</li>
* <li>Snowball original Porter stemmer</li>
* <li>Snowball English stemmer, commonly referred to as Porter2</li>
* <li>Radixor direct dictionary lookup</li>
* <li>Snowball Porter</li>
* <li>Snowball English (Porter2)</li>
* <li>Lucene direct Porter API (generated copy)</li>
* <li>Lucene Porter, KStem, and EnglishMinimal token-filter paths</li>
* <li>Benchmark-only Paice/Husk Lancaster baseline</li>
* </ul>
*
* <p>
* This benchmark compares throughput on a shared workload. It does not imply
* that the algorithms are linguistically equivalent.
* </p>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 1)
@Measurement(iterations = 5, time = 1)
@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
public class EnglishStemmerComparisonBenchmark {
/**
* Shared benchmark data.
* Shared, parameterized benchmark corpus state.
*/
@State(Scope.Benchmark)
public static class SharedState {
/**
* Number of generated lexical families.
*/
@Param({ "1000", "5000" })
public int familyCount;
/**
* Token workload processed by all compared stemmers.
* Shared deterministic token corpus.
*/
private String[] tokens;
/**
* Radixor trie loaded from the bundled professional English dictionary.
* Radixor benchmark adapter for the US/UK benchmark corpus.
*/
private FrequencyTrie<String> radixorTrie;
private RadixorBenchmarkStemmer radixorStemmer;
/**
* Initializes the shared benchmark state.
*
* @throws IOException if the bundled Radixor dictionary cannot be loaded
* Initializes shared corpus and trie state once per trial.
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
this.tokens = EnglishComparisonCorpus.createTokens(this.familyCount);
this.radixorTrie = StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK, true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
public void setUp() throws java.io.IOException {
Logger.getLogger(StemmerDictionaryParser.class.getName())
.setLevel(java.util.logging.Level.OFF);
Logger.getLogger(StemmerDictionaryParser.class.getName()).setUseParentHandlers(false);
this.tokens = EnglishComparisonCorpus.createTokens();
this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.US_UK, true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
}
}
/**
* Per-thread reusable Snowball stemmers.
* Reusable direct stemmer instances.
*/
@State(Scope.Thread)
public static class SnowballState {
public static class DirectStemmerState {
/**
* Adapter for the original Porter stemmer.
* Snowball classic Porter.
*/
private SnowballStemmerAdapter porterStemmer;
/**
* Adapter for the Snowball English stemmer.
* Snowball English (Porter2) for legacy and dictionary comparison.
*/
private SnowballStemmerAdapter englishStemmer;
private SnowballStemmerAdapter englishPorterStemmer;
/**
* Initializes reusable Snowball stemmers for the executing thread.
* Generated Lucene direct Porter implementation copy.
*/
private LucenePorterStemmerCopied lucenePorter;
/**
* Benchmark-only Paice/Husk Lancaster implementation.
*/
private PaiceHuskLancasterStemmer paiceHuskLancaster;
/**
* Apache OpenNLP Porter stemmer.
*/
private opennlp.tools.stemmer.PorterStemmer openNlpPorterStemmer;
/**
* Initializes mutable stemmer instances reused by all benchmark calls.
*/
@Setup(Level.Trial)
public void setUp() {
this.porterStemmer = new SnowballStemmerAdapter(porterStemmer::new);
this.englishStemmer = new SnowballStemmerAdapter(englishStemmer::new);
this.englishPorterStemmer = new SnowballStemmerAdapter(englishStemmer::new);
this.lucenePorter = new LucenePorterStemmerCopied();
this.paiceHuskLancaster = new PaiceHuskLancasterStemmer();
this.openNlpPorterStemmer = new opennlp.tools.stemmer.PorterStemmer();
}
}
/**
* Reusable Lucene token streams and filters.
*/
@State(Scope.Thread)
public static class LuceneFilterState {
/**
* Reused Porter filter stream input.
*/
private final EnglishStemmerComparisonTokenStream porterStemFilterInput;
/**
* Porter token filter for public API integration-path comparison.
*/
private final PorterStemFilter porterStemFilter;
/**
* Porter filter attributes.
*/
private final CharTermAttribute porterStemFilterTerm;
/**
* Reused KStem filter stream input.
*/
private final EnglishStemmerComparisonTokenStream kStemFilterInput;
/**
* KStem token filter for a second Lucene English baseline.
*/
private final KStemFilter kStemFilter;
/**
* KStem filter attributes.
*/
private final CharTermAttribute kStemTerm;
/**
* Reused minimal stem filter stream input.
*/
private final EnglishStemmerComparisonTokenStream englishMinimalStemFilterInput;
/**
* EnglishMinimal token filter.
*/
private final EnglishMinimalStemFilter englishMinimalStemFilter;
/**
* EnglishMinimal filter attributes.
*/
private final CharTermAttribute englishMinimalTerm;
/**
* Reused English possessive filter stream input.
*/
private final EnglishStemmerComparisonTokenStream englishPossessiveFilterInput;
/**
* English possessive filter.
*/
private final EnglishPossessiveFilter englishPossessiveFilter;
/**
* English possessive filter attributes.
*/
private final CharTermAttribute englishPossessiveTerm;
/**
* Creates benchmark stream/filter state and attaches token attributes.
*/
public LuceneFilterState() {
this.porterStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]);
this.porterStemFilter = new PorterStemFilter(this.porterStemFilterInput);
this.porterStemFilterTerm = this.porterStemFilter.getAttribute(CharTermAttribute.class);
this.kStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]);
this.kStemFilter = new KStemFilter(this.kStemFilterInput);
this.kStemTerm = this.kStemFilter.getAttribute(CharTermAttribute.class);
this.englishMinimalStemFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]);
this.englishMinimalStemFilter = new EnglishMinimalStemFilter(this.englishMinimalStemFilterInput);
this.englishMinimalTerm = this.englishMinimalStemFilter.getAttribute(CharTermAttribute.class);
this.englishPossessiveFilterInput = new EnglishStemmerComparisonTokenStream(new String[0]);
this.englishPossessiveFilter = new EnglishPossessiveFilter(this.englishPossessiveFilterInput);
this.englishPossessiveTerm = this.englishPossessiveFilter.getAttribute(CharTermAttribute.class);
}
/**
* Rebinds the shared corpus and resets all streams for another measured
* operation.
*
* <p>
* The {@code String[]} to Lucene character-buffer conversion is deliberately
* performed every time so TokenFilter benchmarks include the cost of adapting
* the benchmark's canonical string corpus to Lucene's mutable token
* attributes.
* </p>
*
* @param tokens benchmark token corpus
*/
void configure(final String[] tokens) throws IOException {
this.porterStemFilterInput.setTokens(tokens);
this.kStemFilterInput.setTokens(tokens);
this.englishMinimalStemFilterInput.setTokens(tokens);
this.englishPossessiveFilterInput.setTokens(tokens);
this.porterStemFilter.reset();
this.kStemFilter.reset();
this.englishMinimalStemFilter.reset();
this.englishPossessiveFilter.reset();
}
/**
* Reuses one mutable filter stream and returns all emitted tokens to blackhole.
*
* @param stream benchmark token stream with configured filter
* @param term token text attribute
* @param blackhole sink
* @throws IOException on token stream failure
*/
private static void consume(final TokenStream stream, final CharTermAttribute term, final Blackhole blackhole)
throws IOException {
while (stream.incrementToken()) {
blackhole.consume(term.toString());
}
stream.end();
}
/**
* Executes Porter filter over the shared corpus.
*
* @param blackhole sink
* @throws IOException if tokenization fails
*/
void runPorterStemFilter(final Blackhole blackhole) throws IOException {
consume(this.porterStemFilter, this.porterStemFilterTerm, blackhole);
}
/**
* Executes KStem filter over the shared corpus.
*
* @param blackhole sink
* @throws IOException if tokenization fails
*/
void runKStemFilter(final Blackhole blackhole) throws IOException {
consume(this.kStemFilter, this.kStemTerm, blackhole);
}
/**
* Executes English minimal filter over the shared corpus.
*
* @param blackhole sink
* @throws IOException if tokenization fails
*/
void runEnglishMinimalStemFilter(final Blackhole blackhole) throws IOException {
consume(this.englishMinimalStemFilter, this.englishMinimalTerm, blackhole);
}
/**
* Executes English possessive filter over the shared corpus.
*
* @param blackhole sink
* @throws IOException if tokenization fails
*/
void runEnglishPossessiveFilter(final Blackhole blackhole) throws IOException {
consume(this.englishPossessiveFilter, this.englishPossessiveTerm, blackhole);
}
}
/**
* Measures Radixor preferred-result stemming throughput.
*
* @param sharedState shared benchmark data
* @param blackhole sink preventing dead-code elimination
* <p>
* This path uses a single shared dictionary lookup and patch application.
* </p>
*
* @param sharedState shared corpus and trie
* @param blackhole result sink
*/
@Benchmark
public void radixorUsUkProfiPreferredStem(final SharedState sharedState, final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final FrequencyTrie<String> trie = sharedState.radixorTrie;
for (String token : tokens) {
final String patch = trie.get(token);
final String stem = patch == null ? token : PatchCommandEncoder.apply(token, patch);
blackhole.consume(stem);
}
}
/**
* Measures Snowball original Porter stemming throughput.
*
* @param sharedState shared benchmark data
* @param snowballState reusable Snowball stemmers
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void snowballOriginalPorter(final SharedState sharedState, final SnowballState snowballState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final SnowballStemmerAdapter stemmer = snowballState.porterStemmer;
final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
@@ -173,25 +347,169 @@ public class EnglishStemmerComparisonBenchmark {
}
/**
* Measures Snowball English stemming throughput.
* Measures the canonical Snowball Porter stemming throughput used by the
* performance badge.
*
* <p>
* Snowball English is the newer English stemmer commonly referred to as
* Porter2.
* This uses Snowball classic Porter as a direct stemmer API call and includes
* no Lucene token stream integration overhead.
* </p>
*
* @param sharedState shared benchmark data
* @param snowballState reusable Snowball stemmers
* @param blackhole sink preventing dead-code elimination
* @param sharedState shared corpus
* @param stemmerState reusable Snowball adapter state
* @param blackhole result sink
*/
@Benchmark
public void snowballEnglishPorter2(final SharedState sharedState, final SnowballState snowballState,
public void snowballOriginalPorter(final SharedState sharedState, final DirectStemmerState stemmerState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final SnowballStemmerAdapter stemmer = snowballState.englishStemmer;
final SnowballStemmerAdapter stemmer = stemmerState.porterStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Measures Snowball English (Porter2) direct API throughput.
*
* @param sharedState shared corpus
* @param stemmerState reusable Snowball adapter state
* @param blackhole result sink
*/
@Benchmark
public void snowballEnglishPorter2(final SharedState sharedState, final DirectStemmerState stemmerState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final SnowballStemmerAdapter stemmer = stemmerState.englishPorterStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Measures Lucene generated Porter stemmer API throughput.
*
* <p>
* This path is a generated copy of Lucene&apos;s package-private PorterStemmer
* class, compiled into the JMH source set only.
* </p>
*
* @param sharedState shared corpus
* @param stemmerState reusable Lucene copied API state
* @param blackhole result sink
*/
@Benchmark
public void lucenePorterStemmerCopied(final SharedState sharedState, final DirectStemmerState stemmerState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final LucenePorterStemmerCopied stemmer = stemmerState.lucenePorter;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Measures Lucene Porter token-filter integration throughput.
*
* <p>
* This includes stream, reusable token attributes, and filter overhead and is
* not equivalent to a direct API stemmer call.
* </p>
*
* @param sharedState shared corpus
* @param filterState reusable filter state
* @param blackhole sink
* @throws IOException if token stream fails
*/
@Benchmark
public void lucenePorterStemFilter(final SharedState sharedState, final LuceneFilterState filterState,
final Blackhole blackhole) throws IOException {
filterState.configure(sharedState.tokens);
filterState.runPorterStemFilter(blackhole);
}
/**
* Measures Lucene KStem integration-path throughput.
*
* @param sharedState shared corpus
* @param filterState reusable filter state
* @param blackhole sink
* @throws IOException if token stream fails
*/
@Benchmark
public void luceneKStemFilter(final SharedState sharedState, final LuceneFilterState filterState,
final Blackhole blackhole) throws IOException {
filterState.configure(sharedState.tokens);
filterState.runKStemFilter(blackhole);
}
/**
* Measures Lucene EnglishMinimal integration-path throughput.
*
* @param sharedState shared corpus
* @param filterState reusable filter state
* @param blackhole sink
* @throws IOException if token stream fails
*/
@Benchmark
public void luceneEnglishMinimalStemFilter(final SharedState sharedState, final LuceneFilterState filterState,
final Blackhole blackhole) throws IOException {
filterState.configure(sharedState.tokens);
filterState.runEnglishMinimalStemFilter(blackhole);
}
/**
* Measures benchmark-only Paice/Husk Lancaster throughput.
*
* @param sharedState shared corpus
* @param stemmerState reusable Paice/Husk instance
* @param blackhole sink
*/
@Benchmark
public void paiceHuskLancaster(final SharedState sharedState, final DirectStemmerState stemmerState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final PaiceHuskLancasterStemmer stemmer = stemmerState.paiceHuskLancaster;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Measures Apache OpenNLP Porter stemming throughput.
*
* @param sharedState shared corpus
* @param stemmerState reusable OpenNLP Porter instance
* @param blackhole sink
*/
@Benchmark
public void opennlpPorterStemmer(final SharedState sharedState, final DirectStemmerState stemmerState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final opennlp.tools.stemmer.PorterStemmer stemmer = stemmerState.openNlpPorterStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token).toString());
}
}
/**
* Measures Lucene EnglishPossessiveFilter as a narrow possessive-removal
* baseline.
*
* @param sharedState shared corpus
* @param filterState reusable filter state
* @param blackhole sink
* @throws IOException if token stream fails
*/
@Benchmark
public void luceneEnglishPossessiveFilter(final SharedState sharedState, final LuceneFilterState filterState,
final Blackhole blackhole) throws IOException {
filterState.configure(sharedState.tokens);
filterState.runEnglishPossessiveFilter(blackhole);
}
}

258
src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmarkQuality.java Normal file
View File

@@ -0,0 +1,258 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer;
import org.openjdk.jmh.annotations.AuxCounters;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
/**
* Emits exact-root agreement metrics for the canonical English badge pair.
*
* <p>
* This class is deliberately named so the existing focused include pattern for
* English stemmer comparison benchmarks includes it. The benchmark methods are
* separate from throughput methods so equality checks do not contaminate timing
* scores.
* </p>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 0)
@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS)
@Fork(0)
public class EnglishStemmerComparisonBenchmarkQuality {
/**
* Shared English quality corpus and stemmer state.
*/
@State(Scope.Benchmark)
public static class SharedState {
/**
* Complete English resource-derived corpus.
*/
private LanguageBenchmarkCorpus.Corpus corpus;
/**
* Compiled Radixor English trie.
*/
private RadixorBenchmarkStemmer radixorStemmer;
/**
* Reusable Snowball Porter adapter.
*/
private SnowballStemmerAdapter porterStemmer;
/**
* Initializes quality resources.
*
* @throws IOException if corpus or trie loading fails
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
this.corpus = LanguageBenchmarkCorpus.createFullCorpus(StemmerPatchTrieLoader.Language.US_UK);
this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.US_UK, true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
this.porterStemmer = new SnowballStemmerAdapter(porterStemmer::new);
}
}
/**
* JMH auxiliary counters for exact-root agreement.
*/
@State(Scope.Thread)
@AuxCounters(AuxCounters.Type.EVENTS)
public static class AccuracyCounters {
/**
* Number of exact-root matches.
*/
public long correctMatches;
/**
* Number of evaluated tokens.
*/
public long evaluatedTokens;
/**
* Number of exact-root matches where the input token differs from the
* expected root.
*/
public long changedCorrectMatches;
/**
* Number of evaluated tokens where the input token differs from the expected
* root.
*/
public long changedEvaluatedTokens;
/**
* Number of exact-root matches where the input token is already the expected
* root.
*/
public long rootPreservedMatches;
/**
* Number of evaluated tokens where the input token is already the expected
* root.
*/
public long rootEvaluatedTokens;
/**
* Resets counters before each measured iteration.
*/
@Setup(Level.Iteration)
public void reset() {
this.correctMatches = 0L;
this.evaluatedTokens = 0L;
this.changedCorrectMatches = 0L;
this.changedEvaluatedTokens = 0L;
this.rootPreservedMatches = 0L;
this.rootEvaluatedTokens = 0L;
}
}
/**
* Evaluates exact-root agreement for the canonical Radixor badge method.
*
* @param sharedState shared English quality state
* @param counters JMH auxiliary counters
* @param blackhole result sink
* @return exact-root match count
*/
@Benchmark
public int radixorUsUkProfiPreferredStemAccuracy(final SharedState sharedState,
final AccuracyCounters counters, final Blackhole blackhole) {
return evaluate(sharedState.corpus, sharedState.radixorStemmer::stem, counters, blackhole);
}
/**
* Evaluates exact-root agreement for the canonical Snowball Porter badge
* method.
*
* @param sharedState shared English quality state
* @param counters JMH auxiliary counters
* @param blackhole result sink
* @return exact-root match count
*/
@Benchmark
public int snowballOriginalPorterAccuracy(final SharedState sharedState,
final AccuracyCounters counters, final Blackhole blackhole) {
return evaluate(sharedState.corpus, sharedState.porterStemmer::stem, counters, blackhole);
}
/**
* Evaluates one stemmer against the expected roots.
*
* @param corpus token/root corpus
* @param stemmer stemmer under evaluation
* @param counters JMH auxiliary counters
* @param blackhole result sink
* @return exact-root match count
*/
private static int evaluate(final LanguageBenchmarkCorpus.Corpus corpus, final Stemmer stemmer,
final AccuracyCounters counters, final Blackhole blackhole) {
Objects.requireNonNull(corpus, "corpus");
Objects.requireNonNull(stemmer, "stemmer");
int correct = 0;
int changedCorrect = 0;
int changedEvaluated = 0;
int rootPreserved = 0;
int rootEvaluated = 0;
final String[] tokens = corpus.tokens();
final String[] expectedRoots = corpus.expectedRoots();
for (int index = 0; index < tokens.length; index++) {
final String token = tokens[index];
final String expectedRoot = expectedRoots[index];
final String actual = stemmer.stem(token);
blackhole.consume(actual);
final boolean exact = Objects.equals(expectedRoot, actual);
if (exact) {
correct++;
}
if (Objects.equals(token, expectedRoot)) {
rootEvaluated++;
if (exact) {
rootPreserved++;
}
} else {
changedEvaluated++;
if (exact) {
changedCorrect++;
}
}
}
counters.correctMatches += correct;
counters.evaluatedTokens += tokens.length;
counters.changedCorrectMatches += changedCorrect;
counters.changedEvaluatedTokens += changedEvaluated;
counters.rootPreservedMatches += rootPreserved;
counters.rootEvaluatedTokens += rootEvaluated;
return correct;
}
/**
* Direct stemmer function.
*/
@FunctionalInterface
private interface Stemmer {
/**
* Produces one stem.
*
* @param token input token
* @return produced stem
*/
String stem(String token);
}
}

145
src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java Normal file
View File

@@ -0,0 +1,145 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
/**
* Reusable token stream driven by a deterministic token corpus.
*
* <p>
* The stream emits each token from a shared array and supports repeated
* {@link #reset()} + {@link #incrementToken()} cycles without per-token
* object allocation.
* </p>
*/
final class EnglishStemmerComparisonTokenStream extends TokenStream {
/**
* Current token text.
*/
private final CharTermAttribute charTermAttribute;
/**
* Token offsets for benchmark stream compliance.
*/
private final OffsetAttribute offsetAttribute;
/**
* Position increment attribute for benchmark stream compliance.
*/
private final PositionIncrementAttribute positionIncrementAttribute;
/**
* Reusable token source.
*/
private final BenchmarkTokenSequence tokenSequence;
/**
* Creates a deterministic token stream for benchmark reuse.
*
* @param tokens tokens emitted by the stream
*/
EnglishStemmerComparisonTokenStream(final String[] tokens) {
this.tokenSequence = new BenchmarkTokenSequence(tokens);
this.charTermAttribute = addAttribute(CharTermAttribute.class);
this.offsetAttribute = addAttribute(OffsetAttribute.class);
this.positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
}
/**
* Replaces the token corpus for this stream.
*
* @param tokens new token corpus
*/
void setTokens(final String[] tokens) {
this.tokenSequence.setTokens(tokens);
}
/**
* Returns whether the stream is drained and ready to be exhausted.
*
* @return true if all configured tokens were consumed
*/
boolean isDrained() {
return !this.tokenSequence.hasNext();
}
/**
* {@inheritDoc}
*/
@Override
public boolean incrementToken() throws IOException {
if (!this.tokenSequence.advance()) {
return false;
}
clearAttributes();
final char[] token = this.tokenSequence.currentToken();
this.charTermAttribute.copyBuffer(token, 0, token.length);
this.positionIncrementAttribute.setPositionIncrement(1);
this.offsetAttribute.setOffset(this.tokenSequence.currentStartOffset(), this.tokenSequence.currentEndOffset());
return true;
}
/**
* {@inheritDoc}
*/
@Override
public void reset() throws IOException {
super.reset();
this.tokenSequence.reset();
}
/**
* {@inheritDoc}
*/
@Override
public void end() throws IOException {
super.end();
final int endOffset = this.tokenSequence.endOffset();
this.offsetAttribute.setOffset(endOffset, endOffset);
}
/**
* {@inheritDoc}
*/
@Override
public void close() throws IOException {
super.close();
this.charTermAttribute.setEmpty();
}
}

1
src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java
View File

@@ -65,6 +65,7 @@ import org.openjdk.jmh.infra.Blackhole;
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 1)
@Measurement(iterations = 5, time = 1)
@SuppressWarnings("deprecation")
public class FrequencyTrieLookupBenchmark {
/**

445
src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java Normal file
View File

@@ -0,0 +1,445 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Locale;
import java.util.Objects;
import java.util.zip.GZIPInputStream;
import org.egothor.stemmer.StemmerPatchTrieLoader;
/**
* Builds deterministic language-specific benchmark corpora from bundled
* Radixor dictionary resources.
*
* <p>
* Corpus construction is setup work only. It is intentionally based on the same
* resource that backs the Radixor benchmark path so every competitor for a
* language consumes the same changed-token timing workload, while quality
* benchmarks can still use the complete dictionary workload.
* </p>
*/
final class LanguageBenchmarkCorpus {
/**
* Minimum token count for timing benchmark operations.
*/
static final int MINIMUM_TIMING_TOKEN_COUNT = 5_000;
/**
* Shared timing corpora keyed by bundled Radixor language.
*/
private static final Map<StemmerPatchTrieLoader.Language, Corpus> TIMING_CORPORA =
new EnumMap<>(StemmerPatchTrieLoader.Language.class);
/**
* Shared changed-token timing corpora keyed by bundled Radixor language.
*/
private static final Map<StemmerPatchTrieLoader.Language, Corpus> CHANGED_TIMING_CORPORA =
new EnumMap<>(StemmerPatchTrieLoader.Language.class);
/**
* Shared complete corpora keyed by bundled Radixor language.
*/
private static final Map<StemmerPatchTrieLoader.Language, Corpus> FULL_CORPORA =
new EnumMap<>(StemmerPatchTrieLoader.Language.class);
/**
* Utility class.
*/
private LanguageBenchmarkCorpus() {
throw new AssertionError("No instances.");
}
/**
* Creates a deterministic changed-token timing corpus from a bundled language
* dictionary.
*
* <p>
* Only token/root pairs where the token differs from the expected root are
* included. Smaller changed-token resources are repeated in stable order until
* the timing corpus reaches 5,000 tokens.
* </p>
*
* @param language bundled Radixor language
* @return token array containing changed-token dictionary entries, repeated
* only when the changed-token resource is smaller than 5,000 tokens
* @throws IOException if the resource cannot be read
*/
static String[] createTokens(final StemmerPatchTrieLoader.Language language) throws IOException {
return createChangedCorpus(language).tokens();
}
/**
* Creates a deterministic changed-token timing corpus from a bundled language
* dictionary.
*
* @param language bundled Radixor language
* @return changed-token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
static Corpus createChangedCorpus(final StemmerPatchTrieLoader.Language language) throws IOException {
return cachedChangedCorpus(language);
}
/**
* Creates a deterministic full-dictionary timing corpus and expected root
* array from a bundled language dictionary.
*
* @param language bundled Radixor language
* @return token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
static Corpus createCorpus(final StemmerPatchTrieLoader.Language language) throws IOException {
return cachedCorpus(TIMING_CORPORA, language, true);
}
/**
* Creates a deterministic full-dictionary timing corpus and expected root
* array from a bundled language dictionary.
*
* <p>
* The complete dictionary token sequence is used when it contains at least
* {@code minimumTokenCount} tokens. Smaller resources are repeated in stable
* order until the minimum is reached.
* </p>
*
* @param language bundled Radixor language
* @param minimumTokenCount minimum token count for timing
* @return token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
static Corpus createCorpus(final StemmerPatchTrieLoader.Language language, final int minimumTokenCount)
throws IOException {
Objects.requireNonNull(language, "language");
if (minimumTokenCount < 1) {
throw new IllegalArgumentException("minimumTokenCount must be at least 1.");
}
if (minimumTokenCount == MINIMUM_TIMING_TOKEN_COUNT) {
return createCorpus(language);
}
return buildTimingCorpus(language, minimumTokenCount);
}
/**
* Creates or returns the shared complete corpus for a bundled language.
*
* @param language bundled Radixor language
* @return complete token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
static Corpus createFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException {
return cachedCorpus(FULL_CORPORA, language, false);
}
/**
* Returns a cached corpus, creating it once per JVM when necessary.
*
* @param cache corpus cache
* @param language bundled Radixor language
* @param timing whether the timing-minimum corpus should be built
* @return cached corpus instance
* @throws IOException if the resource cannot be read
*/
private static Corpus cachedCorpus(final Map<StemmerPatchTrieLoader.Language, Corpus> cache,
final StemmerPatchTrieLoader.Language language, final boolean timing) throws IOException {
Objects.requireNonNull(cache, "cache");
Objects.requireNonNull(language, "language");
synchronized (LanguageBenchmarkCorpus.class) {
final Corpus existing = cache.get(language);
if (existing != null) {
return existing;
}
final Corpus created = timing ? buildTimingCorpus(language, MINIMUM_TIMING_TOKEN_COUNT)
: buildFullCorpus(language);
cache.put(language, created);
return created;
}
}
/**
* Returns a cached changed-token timing corpus, creating it once per JVM when
* necessary.
*
* @param language bundled Radixor language
* @return changed-token timing corpus
* @throws IOException if the resource cannot be read
*/
private static Corpus cachedChangedCorpus(final StemmerPatchTrieLoader.Language language) throws IOException {
Objects.requireNonNull(language, "language");
synchronized (LanguageBenchmarkCorpus.class) {
final Corpus existing = CHANGED_TIMING_CORPORA.get(language);
if (existing != null) {
return existing;
}
final Corpus created = buildChangedTimingCorpus(language, MINIMUM_TIMING_TOKEN_COUNT);
CHANGED_TIMING_CORPORA.put(language, created);
return created;
}
}
/**
* Builds a deterministic timing corpus from a bundled language dictionary.
*
* @param language bundled Radixor language
* @param minimumTokenCount minimum token count for timing
* @return token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
private static Corpus buildTimingCorpus(final StemmerPatchTrieLoader.Language language, final int minimumTokenCount)
throws IOException {
Objects.requireNonNull(language, "language");
if (minimumTokenCount < 1) {
throw new IllegalArgumentException("minimumTokenCount must be at least 1.");
}
final List<Entry> candidates = readCandidates(language, Integer.MAX_VALUE);
if (candidates.isEmpty()) {
throw new IllegalStateException("No benchmark corpus tokens were available for " + language + ".");
}
final int timingTokenCount = Math.max(candidates.size(), minimumTokenCount);
final String[] tokens = new String[timingTokenCount];
final String[] expectedRoots = new String[timingTokenCount];
for (int index = 0; index < tokens.length; index++) {
final Entry entry = candidates.get(index % candidates.size());
tokens[index] = entry.token();
expectedRoots[index] = entry.root();
}
return new Corpus(tokens, expectedRoots);
}
/**
* Builds a deterministic changed-token timing corpus from a bundled language
* dictionary.
*
* @param language bundled Radixor language
* @param minimumTokenCount minimum token count for timing
* @return changed-token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
private static Corpus buildChangedTimingCorpus(final StemmerPatchTrieLoader.Language language,
final int minimumTokenCount) throws IOException {
Objects.requireNonNull(language, "language");
if (minimumTokenCount < 1) {
throw new IllegalArgumentException("minimumTokenCount must be at least 1.");
}
final List<Entry> allCandidates = readCandidates(language, Integer.MAX_VALUE);
final List<Entry> changedCandidates = new ArrayList<>(allCandidates.size());
for (Entry entry : allCandidates) {
if (!Objects.equals(entry.token(), entry.root())) {
changedCandidates.add(entry);
}
}
if (changedCandidates.isEmpty()) {
throw new IllegalStateException("No changed-token benchmark corpus tokens were available for "
+ language + ".");
}
final int timingTokenCount = Math.max(changedCandidates.size(), minimumTokenCount);
final String[] tokens = new String[timingTokenCount];
final String[] expectedRoots = new String[timingTokenCount];
for (int index = 0; index < tokens.length; index++) {
final Entry entry = changedCandidates.get(index % changedCandidates.size());
tokens[index] = entry.token();
expectedRoots[index] = entry.root();
}
return new Corpus(tokens, expectedRoots);
}
/**
* Creates a complete deterministic token corpus and expected root array from a
* bundled language dictionary.
*
* <p>
* This method is intended for exact-root quality accounting. It includes all
* single-token fields available in the dictionary resource and does not repeat
* small dictionaries to the timing minimum.
* </p>
*
* @param language bundled Radixor language
* @return complete token corpus with expected roots
* @throws IOException if the resource cannot be read
*/
private static Corpus buildFullCorpus(final StemmerPatchTrieLoader.Language language) throws IOException {
Objects.requireNonNull(language, "language");
final List<Entry> candidates = readCandidates(language, Integer.MAX_VALUE);
if (candidates.isEmpty()) {
throw new IllegalStateException("No benchmark corpus tokens were available for " + language + ".");
}
final String[] tokens = new String[candidates.size()];
final String[] expectedRoots = new String[candidates.size()];
for (int index = 0; index < tokens.length; index++) {
final Entry entry = candidates.get(index);
tokens[index] = entry.token();
expectedRoots[index] = entry.root();
}
return new Corpus(tokens, expectedRoots);
}
/**
* Reads token candidates from a bundled compressed dictionary.
*
* @param language bundled Radixor language
* @param maximumTokenCount maximum token count to read
* @return deterministic candidate list
* @throws IOException if the resource cannot be read
*/
private static List<Entry> readCandidates(final StemmerPatchTrieLoader.Language language, final int maximumTokenCount)
throws IOException {
final String resourcePath = language.resourcePath();
final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath);
if (resource == null) {
throw new IllegalStateException("Missing bundled benchmark resource " + resourcePath + ".");
}
final List<Entry> candidates = new ArrayList<>(MINIMUM_TIMING_TOKEN_COUNT);
try (InputStream inputStream = resource;
GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
InputStreamReader inputStreamReader = new InputStreamReader(gzipInputStream, StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(inputStreamReader)) {
String line = reader.readLine();
while (line != null && candidates.size() < maximumTokenCount) {
collectLineCandidates(line, candidates, maximumTokenCount);
line = reader.readLine();
}
}
return candidates;
}
/**
* Collects lower-case token candidates from one dictionary line.
*
* @param line dictionary line
* @param candidates mutable candidate list
* @param maximumTokenCount maximum token count to read
*/
private static void collectLineCandidates(final String line, final List<Entry> candidates,
final int maximumTokenCount) {
if (line == null || line.isBlank() || line.startsWith("#") || line.startsWith("//")) {
return;
}
final String[] fields = line.split("\t");
if (fields.length == 0) {
return;
}
final String root = normalizeToken(fields[0]);
if (root.isEmpty() || containsWhitespace(root)) {
return;
}
for (String field : fields) {
if (candidates.size() >= maximumTokenCount) {
return;
}
final String token = normalizeToken(field);
if (!token.isEmpty() && !containsWhitespace(token)) {
candidates.add(new Entry(token, root));
}
}
}
/**
* Normalizes dictionary token text for deterministic benchmark lookup.
*
* @param token dictionary token field
* @return normalized token
*/
private static String normalizeToken(final String token) {
return token.trim().toLowerCase(Locale.ROOT);
}
/**
* Returns whether a token contains Unicode whitespace.
*
* @param token token candidate
* @return {@code true} when whitespace is present
*/
private static boolean containsWhitespace(final String token) {
for (int index = 0; index < token.length(); index++) {
if (Character.isWhitespace(token.charAt(index))) {
return true;
}
}
return false;
}
/**
* Immutable token corpus with expected roots.
*
* @param tokens benchmark token corpus
* @param expectedRoots expected root for each token
*/
record Corpus(String[] tokens, String[] expectedRoots) {
/**
* Creates corpus data.
*
* @param tokens benchmark token corpus
* @param expectedRoots expected root for each token
*/
Corpus {
Objects.requireNonNull(tokens, "tokens");
Objects.requireNonNull(expectedRoots, "expectedRoots");
if (tokens.length != expectedRoots.length) {
throw new IllegalArgumentException("tokens and expectedRoots must have the same length.");
}
}
}
/**
* Immutable dictionary-derived token/root entry.
*
* @param token token form
* @param root expected root
*/
private record Entry(String token, String root) {
}
}

1114
src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java Normal file
View File

File diff suppressed because it is too large Load Diff

82
src/jmh/java/org/egothor/stemmer/benchmark/RadixorBenchmarkStemmer.java Normal file
View File

@@ -0,0 +1,82 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.util.Objects;
import org.egothor.stemmer.CompiledPatchCommand;
import org.egothor.stemmer.FrequencyTrie;
/**
* Benchmark-only Radixor stemmer adapter for the canonical preferred-result
* path over normalized benchmark tokens.
*
* <p>
* The benchmark corpus is normalized during setup, so this adapter uses
* {@link FrequencyTrie#getNormalizedString(String)} to avoid measuring
* redundant lookup-time normalization. Patch commands are applied with the
* traversal direction persisted in the trie metadata.
* </p>
*
* <p>
* Instances are mutable and intended for one JMH worker thread.
* </p>
*/
final class RadixorBenchmarkStemmer {
/**
* Compiled Radixor patch trie with decoded patch-command values.
*/
private final FrequencyTrie<CompiledPatchCommand> trie;
/**
* Creates a benchmark stemmer around one compiled Radixor trie.
*
* @param trie compiled Radixor patch trie
*/
RadixorBenchmarkStemmer(final FrequencyTrie<CompiledPatchCommand> trie) {
this.trie = Objects.requireNonNull(trie, "trie");
}
/**
* Stems one benchmark token through the canonical trie lookup API.
*
* @param token input token
* @return Radixor stem or the input token when no patch is stored
*/
String stem(final String token) {
final CompiledPatchCommand patch = this.trie.getNormalizedString(token);
if (patch == null || patch.preservesAllSources()) {
return token;
}
return patch.apply(token);
}
}

196
src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java Normal file
View File

@@ -0,0 +1,196 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.egothor.stemmer.benchmark.snowball.ext.danishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.dutchStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.finnishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.frenchStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.germanStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.hungarianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.italianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.norwegianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.portugueseStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.russianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.spanishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.swedishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.yiddishStemmer;
/**
* Maps Radixor dictionary languages to matching official Snowball algorithms.
*/
enum SnowballLanguageCase {
/**
* Danish Snowball stemming over the Radixor Danish dictionary.
*/
DANISH("Danish", StemmerPatchTrieLoader.Language.DA_DK, danishStemmer::new, "Danish"),
/**
* Dutch Snowball stemming over the Radixor Dutch dictionary.
*/
DUTCH("Dutch", StemmerPatchTrieLoader.Language.NL_NL, dutchStemmer::new, "Dutch"),
/**
* Finnish Snowball stemming over the Radixor Finnish dictionary.
*/
FINNISH("Finnish", StemmerPatchTrieLoader.Language.FI_FI, finnishStemmer::new, "Finnish"),
/**
* French Snowball stemming over the Radixor French dictionary.
*/
FRENCH("French", StemmerPatchTrieLoader.Language.FR_FR, frenchStemmer::new, "French"),
/**
* German Snowball stemming over the Radixor German dictionary.
*/
GERMAN("German", StemmerPatchTrieLoader.Language.DE_DE, germanStemmer::new, "German"),
/**
* Hungarian Snowball stemming over the Radixor Hungarian dictionary.
*/
HUNGARIAN("Hungarian", StemmerPatchTrieLoader.Language.HU_HU, hungarianStemmer::new, "Hungarian"),
/**
* Italian Snowball stemming over the Radixor Italian dictionary.
*/
ITALIAN("Italian", StemmerPatchTrieLoader.Language.IT_IT, italianStemmer::new, "Italian"),
/**
* Norwegian Snowball stemming over the Radixor Bokmal dictionary.
*/
NORWEGIAN_BOKMAL("Norwegian Bokmal", StemmerPatchTrieLoader.Language.NB_NO, norwegianStemmer::new,
"Norwegian"),
/**
* Norwegian Snowball stemming over the Radixor Nynorsk dictionary.
*/
NORWEGIAN_NYNORSK("Norwegian Nynorsk", StemmerPatchTrieLoader.Language.NN_NO, norwegianStemmer::new,
"Norwegian"),
/**
* Portuguese Snowball stemming over the Radixor Portuguese dictionary.
*/
PORTUGUESE("Portuguese", StemmerPatchTrieLoader.Language.PT_PT, portugueseStemmer::new, "Portuguese"),
/**
* Russian Snowball stemming over the Radixor Russian dictionary.
*/
RUSSIAN("Russian", StemmerPatchTrieLoader.Language.RU_RU, russianStemmer::new, "Russian"),
/**
* Spanish Snowball stemming over the Radixor Spanish dictionary.
*/
SPANISH("Spanish", StemmerPatchTrieLoader.Language.ES_ES, spanishStemmer::new, "Spanish"),
/**
* Swedish Snowball stemming over the Radixor Swedish dictionary.
*/
SWEDISH("Swedish", StemmerPatchTrieLoader.Language.SV_SE, swedishStemmer::new, "Swedish"),
/**
* Yiddish Snowball stemming over the Radixor Yiddish dictionary.
*/
YIDDISH("Yiddish", StemmerPatchTrieLoader.Language.YI, yiddishStemmer::new, "Yiddish");
/**
* Human-readable language name.
*/
private final String displayLanguage;
/**
* Matching Radixor language resource.
*/
private final StemmerPatchTrieLoader.Language radixorLanguage;
/**
* Factory for the isolated benchmark-only Snowball implementation.
*/
private final SnowballStemmerAdapter.Factory directFactory;
/**
* Lucene SnowballFilter algorithm name.
*/
private final String luceneSnowballName;
/**
* Creates a language case.
*
* @param displayLanguage human-readable language name
* @param radixorLanguage matching Radixor language resource
* @param directFactory direct Snowball stemmer factory
* @param luceneSnowballName Lucene SnowballFilter algorithm name
*/
SnowballLanguageCase(final String displayLanguage, final StemmerPatchTrieLoader.Language radixorLanguage,
final SnowballStemmerAdapter.Factory directFactory, final String luceneSnowballName) {
this.displayLanguage = displayLanguage;
this.radixorLanguage = radixorLanguage;
this.directFactory = directFactory;
this.luceneSnowballName = luceneSnowballName;
}
/**
* Returns the human-readable language name.
*
* @return display language
*/
String displayLanguage() {
return this.displayLanguage;
}
/**
* Returns the matching Radixor dictionary language.
*
* @return Radixor language
*/
StemmerPatchTrieLoader.Language radixorLanguage() {
return this.radixorLanguage;
}
/**
* Creates a direct Snowball stemmer adapter.
*
* @return direct Snowball adapter
*/
SnowballStemmerAdapter createDirectStemmer() {
return new SnowballStemmerAdapter(this.directFactory);
}
/**
* Returns the Lucene SnowballFilter algorithm name.
*
* @return Lucene SnowballFilter algorithm name
*/
String luceneSnowballName() {
return this.luceneSnowballName;
}
}

246
src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java Normal file
View File

@@ -0,0 +1,246 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
/**
* Compares Radixor with official Snowball algorithms for every Radixor language
* that has a matching Snowball Java stemmer.
*
* <p>
* Each benchmark operation processes the same changed-token Radixor
* dictionary-derived language corpus, repeated only when the changed-token
* resource contains fewer than 5,000 token fields. The direct Snowball method
* measures the isolated benchmark-only Snowball source. The Lucene
* SnowballFilter method measures Lucene's TokenStream integration path,
* including lower-case normalization and token attribute overhead.
* </p>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
public class SnowballLanguageStemmerComparisonBenchmark {
/**
* Shared language corpus and Radixor trie state.
*/
@State(Scope.Benchmark)
public static class SharedState {
/**
* Language/algorithm case under comparison.
*/
@Param({ "DANISH", "DUTCH", "FINNISH", "FRENCH", "GERMAN", "HUNGARIAN", "ITALIAN",
"NORWEGIAN_BOKMAL", "NORWEGIAN_NYNORSK", "PORTUGUESE", "RUSSIAN", "SPANISH", "SWEDISH",
"YIDDISH" })
public String languageCaseName;
/**
* Resolved language/algorithm case.
*/
private SnowballLanguageCase languageCase;
/**
* Shared deterministic changed-token dictionary corpus.
*/
private String[] tokens;
/**
* Compiled Radixor trie for the selected language.
*/
private RadixorBenchmarkStemmer radixorStemmer;
/**
* Initializes shared language resources before measurement.
*
* @throws IOException if the corpus or trie cannot be loaded
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
this.languageCase = SnowballLanguageCase.valueOf(this.languageCaseName);
this.tokens = LanguageBenchmarkCorpus.createTokens(this.languageCase.radixorLanguage());
this.radixorStemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
this.languageCase.radixorLanguage(), true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
}
}
/**
* Per-thread direct Snowball state.
*/
@State(Scope.Thread)
public static class DirectState {
/**
* Reusable direct Snowball adapter.
*/
private SnowballStemmerAdapter snowballStemmer;
/**
* Initializes direct Snowball state for the selected language.
*
* @param sharedState selected language state
*/
@Setup(Level.Trial)
public void setUp(final SharedState sharedState) {
this.snowballStemmer = sharedState.languageCase.createDirectStemmer();
}
}
/**
* Per-thread Lucene SnowballFilter state.
*/
@State(Scope.Thread)
public static class LuceneSnowballState {
/**
* Reusable benchmark input stream.
*/
private BenchmarkTokenStream input;
/**
* Reusable Lucene SnowballFilter output stream.
*/
private TokenStream output;
/**
* Reusable term attribute.
*/
private CharTermAttribute termAttribute;
/**
* Initializes Lucene SnowballFilter state for the selected language.
*
* @param sharedState selected language state
*/
@Setup(Level.Trial)
public void setUp(final SharedState sharedState) {
this.input = new BenchmarkTokenStream(new String[0]);
final TokenStream normalizedInput = new LowerCaseFilter(this.input);
this.output = new SnowballFilter(normalizedInput, sharedState.languageCase.luceneSnowballName());
this.termAttribute = this.output.addAttribute(CharTermAttribute.class);
}
/**
* Runs the reusable Lucene SnowballFilter over one corpus.
*
* <p>
* The benchmark intentionally rebinds the {@code String[]} corpus on every
* measured operation so the adaptation cost from the canonical string input to
* Lucene's mutable character attributes is included.
* </p>
*
* @param tokens token corpus
* @param blackhole result sink
* @throws IOException if Lucene token streaming fails
*/
void run(final String[] tokens, final Blackhole blackhole) throws IOException {
this.input.setTokens(tokens);
this.output.reset();
while (this.output.incrementToken()) {
blackhole.consume(this.termAttribute.toString());
}
this.output.end();
}
}
/**
* Runs Radixor over the selected Snowball-language corpus.
*
* @param sharedState shared benchmark state
* @param blackhole result sink
*/
@Benchmark
public void radixor(final SharedState sharedState, final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Runs the official Snowball direct Java implementation over the selected
* language corpus.
*
* @param sharedState shared benchmark state
* @param directState reusable direct Snowball state
* @param blackhole result sink
*/
@Benchmark
public void snowballDirect(final SharedState sharedState, final DirectState directState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final SnowballStemmerAdapter stemmer = directState.snowballStemmer;
for (String token : tokens) {
blackhole.consume(stemmer.stem(token));
}
}
/**
* Runs Lucene SnowballFilter over the selected language corpus.
*
* @param sharedState shared benchmark state
* @param luceneState reusable Lucene Snowball state
* @param blackhole result sink
* @throws IOException if Lucene token streaming fails
*/
@Benchmark
public void luceneSnowballFilter(final SharedState sharedState, final LuceneSnowballState luceneState,
final Blackhole blackhole) throws IOException {
luceneState.run(sharedState.tokens, blackhole);
}
}

2
src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java
View File

@@ -32,7 +32,7 @@ package org.egothor.stemmer.benchmark;
import java.util.Objects;
import org.tartarus.snowball.SnowballStemmer;
import org.egothor.stemmer.benchmark.snowball.SnowballStemmer;
/**
* Small adapter around a Snowball stemmer instance used by benchmarks.

813
src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java Normal file
View File

@@ -0,0 +1,813 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.cz.CzechStemFilter;
import org.apache.lucene.analysis.de.GermanLightStemFilter;
import org.apache.lucene.analysis.de.GermanMinimalStemFilter;
import org.apache.lucene.analysis.de.GermanNormalizationFilter;
import org.apache.lucene.analysis.de.GermanStemFilter;
import org.apache.lucene.analysis.en.EnglishMinimalStemFilter;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.KStemFilter;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.es.SpanishLightStemFilter;
import org.apache.lucene.analysis.es.SpanishMinimalStemFilter;
import org.apache.lucene.analysis.es.SpanishPluralStemFilter;
import org.apache.lucene.analysis.fa.PersianNormalizationFilter;
import org.apache.lucene.analysis.fa.PersianStemFilter;
import org.apache.lucene.analysis.fi.FinnishLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter;
import org.apache.lucene.analysis.hu.HungarianLightStemFilter;
import org.apache.lucene.analysis.it.ItalianLightStemFilter;
import org.apache.lucene.analysis.morfologik.MorfologikFilter;
import org.apache.lucene.analysis.no.NorwegianLightStemFilter;
import org.apache.lucene.analysis.no.NorwegianMinimalStemFilter;
import org.apache.lucene.analysis.pl.PolishAnalyzer;
import org.apache.lucene.analysis.pt.PortugueseLightStemFilter;
import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter;
import org.apache.lucene.analysis.pt.PortugueseStemFilter;
import org.apache.lucene.analysis.ru.RussianLightStemFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.stempel.StempelFilter;
import org.apache.lucene.analysis.stempel.StempelStemmer;
import org.apache.lucene.analysis.sv.SwedishLightStemFilter;
import org.apache.lucene.analysis.sv.SwedishMinimalStemFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.egothor.stemmer.benchmark.snowball.ext.englishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.porterStemmer;
import org.openjdk.jmh.annotations.AuxCounters;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import morfologik.stemming.Dictionary;
import morfologik.stemming.DictionaryLookup;
import morfologik.stemming.WordData;
/**
* Emits exact-root agreement metrics through standard JMH result files.
*
* <p>
* This benchmark is a quality pass, not a throughput competitor. Each operation
* evaluates one stemmer against the complete Radixor dictionary resource for
* the matching language. The useful outputs are the JMH auxiliary counters
* {@code correctMatches}, {@code evaluatedTokens},
* {@code changedCorrectMatches}, {@code changedEvaluatedTokens},
* {@code rootPreservedMatches}, and {@code rootEvaluatedTokens}.
* </p>
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 0)
@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS)
@Fork(0)
public class StemmerComparisonBenchmarkQuality {
/**
* Shared quality state for one candidate stemmer.
*/
@State(Scope.Benchmark)
public static class QualityState {
/**
* Candidate stemmer whose exact-root agreement is measured.
*/
@Param({
"ENGLISH_RADIXOR",
"ENGLISH_SNOWBALL_ORIGINAL_PORTER",
"ENGLISH_SNOWBALL_PORTER2",
"ENGLISH_LUCENE_PORTER_COPIED",
"ENGLISH_LUCENE_PORTER_FILTER",
"ENGLISH_LUCENE_KSTEM_FILTER",
"ENGLISH_LUCENE_MINIMAL_FILTER",
"ENGLISH_LUCENE_POSSESSIVE_FILTER",
"ENGLISH_PAICE_HUSK_LANCASTER",
"ENGLISH_OPENNLP_PORTER",
"CZECH_RADIXOR",
"CZECH_LUCENE_CZECH_STEM_FILTER",
"GERMAN_RADIXOR",
"GERMAN_LUCENE_GERMAN_STEM_FILTER",
"GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER",
"GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER",
"SPANISH_RADIXOR",
"SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER",
"SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER",
"SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER",
"PERSIAN_RADIXOR",
"PERSIAN_LUCENE_PERSIAN_STEM_FILTER",
"FINNISH_RADIXOR",
"FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER",
"FRENCH_RADIXOR",
"FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER",
"FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER",
"HUNGARIAN_RADIXOR",
"HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER",
"ITALIAN_RADIXOR",
"ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER",
"NORWEGIAN_BOKMAL_RADIXOR",
"NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER",
"NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER",
"POLISH_RADIXOR",
"POLISH_LUCENE_STEMPEL_DIRECT",
"POLISH_LUCENE_STEMPEL_FILTER",
"POLISH_LUCENE_MORFOLOGIK_FILTER",
"PORTUGUESE_RADIXOR",
"PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER",
"PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER",
"PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER",
"RUSSIAN_RADIXOR",
"RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER",
"SWEDISH_RADIXOR",
"SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER",
"SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER",
"UKRAINIAN_RADIXOR",
"UKRAINIAN_MORFOLOGIK_DIRECT",
"UKRAINIAN_LUCENE_MORFOLOGIK_FILTER",
"SNOWBALL_DANISH_RADIXOR",
"SNOWBALL_DANISH_DIRECT",
"SNOWBALL_DANISH_LUCENE_FILTER",
"SNOWBALL_DUTCH_RADIXOR",
"SNOWBALL_DUTCH_DIRECT",
"SNOWBALL_DUTCH_LUCENE_FILTER",
"SNOWBALL_FINNISH_RADIXOR",
"SNOWBALL_FINNISH_DIRECT",
"SNOWBALL_FINNISH_LUCENE_FILTER",
"SNOWBALL_FRENCH_RADIXOR",
"SNOWBALL_FRENCH_DIRECT",
"SNOWBALL_FRENCH_LUCENE_FILTER",
"SNOWBALL_GERMAN_RADIXOR",
"SNOWBALL_GERMAN_DIRECT",
"SNOWBALL_GERMAN_LUCENE_FILTER",
"SNOWBALL_HUNGARIAN_RADIXOR",
"SNOWBALL_HUNGARIAN_DIRECT",
"SNOWBALL_HUNGARIAN_LUCENE_FILTER",
"SNOWBALL_ITALIAN_RADIXOR",
"SNOWBALL_ITALIAN_DIRECT",
"SNOWBALL_ITALIAN_LUCENE_FILTER",
"SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR",
"SNOWBALL_NORWEGIAN_BOKMAL_DIRECT",
"SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER",
"SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR",
"SNOWBALL_NORWEGIAN_NYNORSK_DIRECT",
"SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER",
"SNOWBALL_PORTUGUESE_RADIXOR",
"SNOWBALL_PORTUGUESE_DIRECT",
"SNOWBALL_PORTUGUESE_LUCENE_FILTER",
"SNOWBALL_RUSSIAN_RADIXOR",
"SNOWBALL_RUSSIAN_DIRECT",
"SNOWBALL_RUSSIAN_LUCENE_FILTER",
"SNOWBALL_SPANISH_RADIXOR",
"SNOWBALL_SPANISH_DIRECT",
"SNOWBALL_SPANISH_LUCENE_FILTER",
"SNOWBALL_SWEDISH_RADIXOR",
"SNOWBALL_SWEDISH_DIRECT",
"SNOWBALL_SWEDISH_LUCENE_FILTER",
"SNOWBALL_YIDDISH_RADIXOR",
"SNOWBALL_YIDDISH_DIRECT",
"SNOWBALL_YIDDISH_LUCENE_FILTER"
})
public String candidateName;
/**
* Full dictionary corpus for the selected language.
*/
private LanguageBenchmarkCorpus.Corpus corpus;
/**
* Candidate evaluator.
*/
private QualityEvaluator evaluator;
/**
* Initializes corpus and evaluator before measurement.
*
* @throws IOException if dictionary or stemmer resources cannot be loaded
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
final QualityCandidate candidate = QualityCandidate.valueOf(this.candidateName);
this.corpus = LanguageBenchmarkCorpus.createFullCorpus(candidate.radixorLanguage());
this.evaluator = candidate.createEvaluator();
}
}
/**
* JMH auxiliary counters for exact-root agreement.
*/
@State(Scope.Thread)
@AuxCounters(AuxCounters.Type.EVENTS)
public static class QualityCounters {
/**
* Number of outputs equal to the dictionary root.
*/
public long correctMatches;
/**
* Number of evaluated input tokens.
*/
public long evaluatedTokens;
/**
* Number of exact-root matches where the input token differs from the
* expected root.
*/
public long changedCorrectMatches;
/**
* Number of evaluated tokens where the input token differs from the expected
* root.
*/
public long changedEvaluatedTokens;
/**
* Number of exact-root matches where the input token is already the expected
* root.
*/
public long rootPreservedMatches;
/**
* Number of evaluated tokens where the input token is already the expected
* root.
*/
public long rootEvaluatedTokens;
/**
* Resets counters before each measured iteration.
*/
@Setup(Level.Iteration)
public void reset() {
this.correctMatches = 0L;
this.evaluatedTokens = 0L;
this.changedCorrectMatches = 0L;
this.changedEvaluatedTokens = 0L;
this.rootPreservedMatches = 0L;
this.rootEvaluatedTokens = 0L;
}
}
/**
* Runs exact-root agreement over the full dictionary corpus.
*
* @param state quality state
* @param counters auxiliary JMH counters
* @param blackhole result sink
* @return exact-root match count for this operation
* @throws IOException if Lucene streaming fails
*/
@Benchmark
public int exactRootAgreement(final QualityState state, final QualityCounters counters, final Blackhole blackhole)
throws IOException {
final QualityResult result = state.evaluator.evaluate(state.corpus, blackhole);
counters.correctMatches += result.correctMatches();
counters.evaluatedTokens += result.evaluatedTokens();
counters.changedCorrectMatches += result.changedCorrectMatches();
counters.changedEvaluatedTokens += result.changedEvaluatedTokens();
counters.rootPreservedMatches += result.rootPreservedMatches();
counters.rootEvaluatedTokens += result.rootEvaluatedTokens();
return result.correctMatches();
}
/**
* Candidate stemmers that can be evaluated against a Radixor resource.
*/
private enum QualityCandidate {
ENGLISH_RADIXOR(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_SNOWBALL_ORIGINAL_PORTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_SNOWBALL_PORTER2(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_LUCENE_PORTER_COPIED(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_LUCENE_PORTER_FILTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_LUCENE_KSTEM_FILTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_LUCENE_MINIMAL_FILTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_LUCENE_POSSESSIVE_FILTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_PAICE_HUSK_LANCASTER(StemmerPatchTrieLoader.Language.US_UK),
ENGLISH_OPENNLP_PORTER(StemmerPatchTrieLoader.Language.US_UK),
CZECH_RADIXOR(StemmerPatchTrieLoader.Language.CS_CZ),
CZECH_LUCENE_CZECH_STEM_FILTER(StemmerPatchTrieLoader.Language.CS_CZ),
GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE),
GERMAN_LUCENE_GERMAN_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES),
SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES),
SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES),
SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES),
PERSIAN_RADIXOR(StemmerPatchTrieLoader.Language.FA_IR),
PERSIAN_LUCENE_PERSIAN_STEM_FILTER(StemmerPatchTrieLoader.Language.FA_IR),
FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI),
FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FI_FI),
FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR),
FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR),
FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.FR_FR),
HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU),
HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.HU_HU),
ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT),
ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.IT_IT),
NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO),
NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO),
NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.NB_NO),
POLISH_RADIXOR(StemmerPatchTrieLoader.Language.PL_PL),
POLISH_LUCENE_STEMPEL_DIRECT(StemmerPatchTrieLoader.Language.PL_PL),
POLISH_LUCENE_STEMPEL_FILTER(StemmerPatchTrieLoader.Language.PL_PL),
POLISH_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.PL_PL),
PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT),
PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT),
PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT),
PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.PT_PT),
RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU),
RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.RU_RU),
SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE),
SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE),
SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.SV_SE),
UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA),
UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA),
UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA),
SNOWBALL_DANISH_RADIXOR(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
SNOWBALL_DUTCH_RADIXOR(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
SNOWBALL_DUTCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
SNOWBALL_FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
SNOWBALL_FINNISH_DIRECT(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
SNOWBALL_FINNISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
SNOWBALL_FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
SNOWBALL_FRENCH_DIRECT(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
SNOWBALL_FRENCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
SNOWBALL_GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
SNOWBALL_GERMAN_DIRECT(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
SNOWBALL_GERMAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
SNOWBALL_HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
SNOWBALL_HUNGARIAN_DIRECT(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
SNOWBALL_HUNGARIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
SNOWBALL_ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
SNOWBALL_ITALIAN_DIRECT(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
SNOWBALL_ITALIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO,
SnowballLanguageCase.NORWEGIAN_BOKMAL),
SNOWBALL_NORWEGIAN_BOKMAL_DIRECT(StemmerPatchTrieLoader.Language.NB_NO,
SnowballLanguageCase.NORWEGIAN_BOKMAL),
SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NB_NO,
SnowballLanguageCase.NORWEGIAN_BOKMAL),
SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR(StemmerPatchTrieLoader.Language.NN_NO,
SnowballLanguageCase.NORWEGIAN_NYNORSK),
SNOWBALL_NORWEGIAN_NYNORSK_DIRECT(StemmerPatchTrieLoader.Language.NN_NO,
SnowballLanguageCase.NORWEGIAN_NYNORSK),
SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO,
SnowballLanguageCase.NORWEGIAN_NYNORSK),
SNOWBALL_PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
SNOWBALL_RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
SNOWBALL_RUSSIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
SNOWBALL_SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
SNOWBALL_SPANISH_DIRECT(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
SNOWBALL_SPANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
SNOWBALL_SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
SNOWBALL_SWEDISH_DIRECT(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
SNOWBALL_SWEDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
SNOWBALL_YIDDISH_RADIXOR(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH),
SNOWBALL_YIDDISH_DIRECT(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH),
SNOWBALL_YIDDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH);
/**
* Radixor dictionary language used as truth.
*/
private final StemmerPatchTrieLoader.Language radixorLanguage;
/**
* Optional Snowball language mapping.
*/
private final SnowballLanguageCase snowballLanguageCase;
/**
* Creates a candidate.
*
* @param radixorLanguage Radixor dictionary language
*/
QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage) {
this(radixorLanguage, null);
}
/**
* Creates a candidate.
*
* @param radixorLanguage Radixor dictionary language
* @param snowballLanguageCase matching Snowball case
*/
QualityCandidate(final StemmerPatchTrieLoader.Language radixorLanguage,
final SnowballLanguageCase snowballLanguageCase) {
this.radixorLanguage = radixorLanguage;
this.snowballLanguageCase = snowballLanguageCase;
}
/**
* Returns the Radixor dictionary language.
*
* @return Radixor language
*/
StemmerPatchTrieLoader.Language radixorLanguage() {
return this.radixorLanguage;
}
/**
* Creates the evaluator for this candidate.
*
* @return quality evaluator
* @throws IOException if stemmer resources cannot be loaded
*/
QualityEvaluator createEvaluator() throws IOException {
if (name().endsWith("_RADIXOR")) {
return direct(createRadixorStemmer(this.radixorLanguage));
}
if (name().endsWith("_DIRECT") && this.snowballLanguageCase != null) {
return direct(this.snowballLanguageCase.createDirectStemmer()::stem);
}
if (name().endsWith("_LUCENE_FILTER") && this.snowballLanguageCase != null) {
return tokenFilter(input -> new SnowballFilter(new LowerCaseFilter(input),
this.snowballLanguageCase.luceneSnowballName()));
}
return switch (this) {
case ENGLISH_SNOWBALL_ORIGINAL_PORTER -> direct(new SnowballStemmerAdapter(porterStemmer::new)::stem);
case ENGLISH_SNOWBALL_PORTER2 -> direct(new SnowballStemmerAdapter(englishStemmer::new)::stem);
case ENGLISH_LUCENE_PORTER_COPIED -> direct(new LucenePorterStemmerCopied()::stem);
case ENGLISH_LUCENE_PORTER_FILTER -> tokenFilter(PorterStemFilter::new);
case ENGLISH_LUCENE_KSTEM_FILTER -> tokenFilter(KStemFilter::new);
case ENGLISH_LUCENE_MINIMAL_FILTER -> tokenFilter(EnglishMinimalStemFilter::new);
case ENGLISH_LUCENE_POSSESSIVE_FILTER -> tokenFilter(EnglishPossessiveFilter::new);
case ENGLISH_PAICE_HUSK_LANCASTER -> direct(new PaiceHuskLancasterStemmer()::stem);
case ENGLISH_OPENNLP_PORTER -> {
final opennlp.tools.stemmer.PorterStemmer stemmer =
new opennlp.tools.stemmer.PorterStemmer();
yield direct(token -> stemmer.stem(token).toString());
}
case CZECH_LUCENE_CZECH_STEM_FILTER -> tokenFilter(input -> new CzechStemFilter(lowercase(input)));
case GERMAN_LUCENE_GERMAN_STEM_FILTER -> tokenFilter(input -> new GermanStemFilter(lowercase(input)));
case GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER ->
tokenFilter(input -> new GermanLightStemFilter(germanNormalize(input)));
case GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new GermanMinimalStemFilter(germanNormalize(input)));
case SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER ->
tokenFilter(input -> new SpanishLightStemFilter(lowercase(input)));
case SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new SpanishMinimalStemFilter(lowercase(input)));
case SPANISH_LUCENE_SPANISH_PLURAL_STEM_FILTER ->
tokenFilter(input -> new SpanishPluralStemFilter(lowercase(input)));
case PERSIAN_LUCENE_PERSIAN_STEM_FILTER ->
tokenFilter(input -> new PersianStemFilter(persianNormalize(input)));
case FINNISH_LUCENE_FINNISH_LIGHT_STEM_FILTER ->
tokenFilter(input -> new FinnishLightStemFilter(lowercase(input)));
case FRENCH_LUCENE_FRENCH_LIGHT_STEM_FILTER ->
tokenFilter(input -> new FrenchLightStemFilter(lowercase(input)));
case FRENCH_LUCENE_FRENCH_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new FrenchMinimalStemFilter(lowercase(input)));
case HUNGARIAN_LUCENE_HUNGARIAN_LIGHT_STEM_FILTER ->
tokenFilter(input -> new HungarianLightStemFilter(lowercase(input)));
case ITALIAN_LUCENE_ITALIAN_LIGHT_STEM_FILTER ->
tokenFilter(input -> new ItalianLightStemFilter(lowercase(input)));
case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_LIGHT_STEM_FILTER ->
tokenFilter(input -> new NorwegianLightStemFilter(lowercase(input)));
case NORWEGIAN_BOKMAL_LUCENE_NORWEGIAN_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new NorwegianMinimalStemFilter(lowercase(input)));
case POLISH_LUCENE_STEMPEL_DIRECT -> {
final StempelStemmer stemmer = new StempelStemmer(PolishAnalyzer.getDefaultTable());
yield direct(token -> {
final StringBuilder stem = stemmer.stem(token);
return stem == null ? token : stem.toString();
});
}
case POLISH_LUCENE_STEMPEL_FILTER ->
tokenFilter(input -> new StempelFilter(input, new StempelStemmer(PolishAnalyzer.getDefaultTable())));
case POLISH_LUCENE_MORFOLOGIK_FILTER -> tokenFilter(MorfologikFilter::new);
case PORTUGUESE_LUCENE_PORTUGUESE_STEM_FILTER ->
tokenFilter(input -> new PortugueseStemFilter(lowercase(input)));
case PORTUGUESE_LUCENE_PORTUGUESE_LIGHT_STEM_FILTER ->
tokenFilter(input -> new PortugueseLightStemFilter(lowercase(input)));
case PORTUGUESE_LUCENE_PORTUGUESE_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new PortugueseMinimalStemFilter(lowercase(input)));
case RUSSIAN_LUCENE_RUSSIAN_LIGHT_STEM_FILTER ->
tokenFilter(input -> new RussianLightStemFilter(lowercase(input)));
case SWEDISH_LUCENE_SWEDISH_LIGHT_STEM_FILTER ->
tokenFilter(input -> new SwedishLightStemFilter(lowercase(input)));
case SWEDISH_LUCENE_SWEDISH_MINIMAL_STEM_FILTER ->
tokenFilter(input -> new SwedishMinimalStemFilter(lowercase(input)));
case UKRAINIAN_MORFOLOGIK_DIRECT -> {
final DictionaryLookup lookup = new DictionaryLookup(loadUkrainianMorfologikDictionary());
yield direct(token -> firstMorfologikStem(token, lookup));
}
case UKRAINIAN_LUCENE_MORFOLOGIK_FILTER -> {
final Dictionary dictionary = loadUkrainianMorfologikDictionary();
yield tokenFilter(input -> new MorfologikFilter(input, dictionary));
}
default -> throw new IllegalStateException("No evaluator for " + this + ".");
};
}
}
/**
* Direct stemmer function.
*/
@FunctionalInterface
private interface Stemmer {
/**
* Produces one stem.
*
* @param token input token
* @return produced stem
*/
String stem(String token);
}
/**
* Quality evaluator for one candidate.
*/
@FunctionalInterface
private interface QualityEvaluator {
/**
* Evaluates exact-root agreement for one corpus.
*
* @param corpus token/root corpus
* @param blackhole result sink
* @return exact-root match count
* @throws IOException if Lucene streaming fails
*/
QualityResult evaluate(LanguageBenchmarkCorpus.Corpus corpus, Blackhole blackhole) throws IOException;
}
/**
* Exact-root agreement counters for one quality operation.
*
* @param correctMatches total exact-root matches
* @param evaluatedTokens total evaluated tokens
* @param changedCorrectMatches exact-root matches where token differs from root
* @param changedEvaluatedTokens evaluated tokens where token differs from root
* @param rootPreservedMatches exact-root matches where token already equals root
* @param rootEvaluatedTokens evaluated tokens where token already equals root
*/
private record QualityResult(int correctMatches, int evaluatedTokens, int changedCorrectMatches,
int changedEvaluatedTokens, int rootPreservedMatches, int rootEvaluatedTokens) {
}
/**
* Creates a direct evaluator.
*
* @param stemmer direct stemmer
* @return quality evaluator
*/
private static QualityEvaluator direct(final Stemmer stemmer) {
Objects.requireNonNull(stemmer, "stemmer");
return (corpus, blackhole) -> {
int correct = 0;
int changedCorrect = 0;
int changedEvaluated = 0;
int rootPreserved = 0;
int rootEvaluated = 0;
final String[] tokens = corpus.tokens();
final String[] expectedRoots = corpus.expectedRoots();
for (int index = 0; index < tokens.length; index++) {
final String token = tokens[index];
final String expectedRoot = expectedRoots[index];
final String actual = stemmer.stem(token);
blackhole.consume(actual);
final boolean exact = Objects.equals(expectedRoot, actual);
if (exact) {
correct++;
}
if (Objects.equals(token, expectedRoot)) {
rootEvaluated++;
if (exact) {
rootPreserved++;
}
} else {
changedEvaluated++;
if (exact) {
changedCorrect++;
}
}
}
return new QualityResult(correct, tokens.length, changedCorrect, changedEvaluated, rootPreserved,
rootEvaluated);
};
}
/**
* Creates a TokenFilter evaluator.
*
* @param factory token stream factory
* @return quality evaluator
*/
private static QualityEvaluator tokenFilter(final Function<TokenStream, TokenStream> factory) {
Objects.requireNonNull(factory, "factory");
return (corpus, blackhole) -> {
final String[] actualStems = firstTokenFilterOutputs(corpus.tokens(), factory, blackhole);
final String[] expectedRoots = corpus.expectedRoots();
final String[] tokens = corpus.tokens();
int correct = 0;
int changedCorrect = 0;
int changedEvaluated = 0;
int rootPreserved = 0;
int rootEvaluated = 0;
for (int index = 0; index < actualStems.length; index++) {
final String token = tokens[index];
final String expectedRoot = expectedRoots[index];
final boolean exact = Objects.equals(expectedRoot, actualStems[index]);
if (exact) {
correct++;
}
if (Objects.equals(token, expectedRoot)) {
rootEvaluated++;
if (exact) {
rootPreserved++;
}
} else {
changedEvaluated++;
if (exact) {
changedCorrect++;
}
}
}
return new QualityResult(correct, actualStems.length, changedCorrect, changedEvaluated, rootPreserved,
rootEvaluated);
};
}
/**
* Creates a direct Radixor stemmer.
*
* @param language Radixor dictionary language
* @return direct stemmer
* @throws IOException if the trie cannot be loaded
*/
private static Stemmer createRadixorStemmer(final StemmerPatchTrieLoader.Language language) throws IOException {
final RadixorBenchmarkStemmer stemmer = new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
language, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
return stemmer::stem;
}
/**
* Loads the benchmark-only Ukrainian Morfologik dictionary.
*
* @return Ukrainian Morfologik dictionary
* @throws IOException if the dictionary cannot be loaded
*/
private static Dictionary loadUkrainianMorfologikDictionary() throws IOException {
final URL dictionaryUrl = StemmerComparisonBenchmarkQuality.class.getClassLoader()
.getResource("ua/net/nlp/ukrainian.dict");
if (dictionaryUrl == null) {
throw new IllegalStateException("Missing Ukrainian Morfologik dictionary resource.");
}
return Dictionary.read(dictionaryUrl);
}
/**
* Returns the first Morfologik stem for one token.
*
* @param token input token
* @param lookup dictionary lookup
* @return first Morfologik stem, or the input token when no analysis exists
*/
private static String firstMorfologikStem(final String token, final DictionaryLookup lookup) {
final List<WordData> analyses = lookup.lookup(token);
if (analyses.isEmpty()) {
return token;
}
return analyses.get(0).getStem().toString();
}
/**
* Extracts the first emitted term for each input token from a TokenFilter
* pipeline.
*
* @param tokens token corpus
* @param factory token stream factory
* @param blackhole result sink
* @return first emitted term per input token
* @throws IOException if Lucene streaming fails
*/
private static String[] firstTokenFilterOutputs(final String[] tokens, final Function<TokenStream, TokenStream> factory,
final Blackhole blackhole) throws IOException {
final String[] outputs = new String[tokens.length];
final BenchmarkTokenStream input = new BenchmarkTokenStream(tokens);
final TokenStream output = factory.apply(input);
final CharTermAttribute termAttribute = output.addAttribute(CharTermAttribute.class);
final PositionIncrementAttribute positionAttribute = output.addAttribute(PositionIncrementAttribute.class);
int inputIndex = -1;
boolean recordedForPosition = false;
output.reset();
while (output.incrementToken()) {
final int positionIncrement = positionAttribute.getPositionIncrement();
if (positionIncrement > 0) {
inputIndex += positionIncrement;
recordedForPosition = false;
}
if (inputIndex >= 0 && inputIndex < outputs.length && !recordedForPosition) {
outputs[inputIndex] = termAttribute.toString();
recordedForPosition = true;
}
blackhole.consume(termAttribute);
}
output.end();
output.close();
for (int index = 0; index < outputs.length; index++) {
if (outputs[index] == null) {
outputs[index] = tokens[index];
}
}
return outputs;
}
/**
* Adds Lucene lower-case normalization.
*
* @param input input token stream
* @return normalized stream
*/
private static TokenStream lowercase(final TokenStream input) {
return new LowerCaseFilter(input);
}
/**
* Adds Lucene German normalization.
*
* @param input input token stream
* @return normalized stream
*/
private static TokenStream germanNormalize(final TokenStream input) {
return new GermanNormalizationFilter(lowercase(input));
}
/**
* Adds Lucene Persian normalization.
*
* @param input input token stream
* @return normalized stream
*/
private static TokenStream persianNormalize(final TokenStream input) {
TokenStream result = lowercase(input);
result = new DecimalDigitFilter(result);
result = new ArabicNormalizationFilter(result);
result = new PersianNormalizationFilter(result);
return result;
}
}

1
src/main/java/org/egothor/stemmer/Compile.java
View File

@@ -147,6 +147,7 @@ public final class Compile {
* @param arguments parsed command-line arguments
* @throws IOException if compilation or output writing fails
*/
@SuppressWarnings("deprecation")
private static void compile(final Arguments arguments) throws IOException {
final ReductionSettings reductionSettings = new ReductionSettings(arguments.reductionMode(),
arguments.dominantWinnerMinPercent(), arguments.dominantWinnerOverSecondRatio());

1282
src/main/java/org/egothor/stemmer/CompiledPatchCommand.java Normal file
View File

File diff suppressed because it is too large Load Diff

240
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -89,7 +89,7 @@ import org.egothor.stemmer.trie.ReductionSignature;
*
* @param <V> value type
*/
@SuppressWarnings("PMD.CyclomaticComplexity")
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.CouplingBetweenObjects" })
public final class FrequencyTrie<V> {
/**
@@ -105,7 +105,7 @@ public final class FrequencyTrie<V> {
/**
* Version of the canonical fingerprint input format.
*/
private static final int FINGERPRINT_FORMAT_VERSION = 1;
private static final int FINGERPRINT_FORMAT_VERSION = 2;
/**
* Root node of the compiled read-only trie.
@@ -169,7 +169,7 @@ public final class FrequencyTrie<V> {
/**
* Binary format version.
*/
private static final int STREAM_VERSION = 5;
private static final int STREAM_VERSION = 6;
/**
* Version where traversal-direction ordinal is persisted.
@@ -186,6 +186,16 @@ public final class FrequencyTrie<V> {
*/
private static final int CASE_VERSION = 4;
/**
* Version where the persisted metadata switched to a text block.
*/
private static final int TEXT_METADATA_VERSION = 5;
/**
* Version where contracted accepting nodes are persisted.
*/
private static final int ACCEPTING_NODE_VERSION = 6;
/**
* Argument name for lookup keys.
*/
@@ -260,6 +270,21 @@ public final class FrequencyTrie<V> {
this.emptyValues = arrayFactory.apply(0);
}
/**
* Creates a trie from an already compiled root.
*
* @param arrayFactory array factory
* @param root compiled root
* @param metadata trie metadata
* @param <V> value type
* @return trie instance
*/
/* default */ static <V> FrequencyTrie<V> fromCompiled(final IntFunction<V[]> arrayFactory,
final CompiledNode<V> root,
final TrieMetadata metadata) {
return new FrequencyTrie<>(arrayFactory, root, metadata);
}
/**
* Returns the most frequent value stored at the node addressed by the supplied
* key.
@@ -292,6 +317,63 @@ public final class FrequencyTrie<V> {
return orderedValues[0];
}
/**
* Returns the preferred value for an already-normalized key.
*
* <p>
* This method bypasses {@link TrieMetadata#caseProcessingMode()} and
* {@link TrieMetadata#diacriticProcessingMode()}. Callers must supply input
* normalized exactly as required by this trie's metadata. It is intended for
* hot paths where normalization is guaranteed by an upstream tokenizer or
* benchmark corpus and repeated lookup-time normalization would be redundant.
* </p>
*
* @param key already-normalized key to resolve
* @return most frequent value, or {@code null} if the key does not exist or no
* value is stored at the addressed node
* @throws NullPointerException if {@code key} is {@code null}
*/
public V getNormalized(final CharSequence key) {
Objects.requireNonNull(key, ARG_KEY);
final CompiledNode<V> node = findNode(key);
if (node == null) {
return null;
}
final V[] orderedValues = node.orderedValues();
if (orderedValues.length == 0) {
return null;
}
return orderedValues[0];
}
/**
* Returns the preferred value for an already-normalized {@link String} key.
*
* <p>
* This overload keeps high-volume string lookup on a monomorphic path and
* avoids the {@link CharSequence} dispatch used by the general overload.
* Callers must supply input normalized exactly as required by this trie's
* metadata.
* </p>
*
* @param key already-normalized key to resolve
* @return most frequent value, or {@code null} if the key does not exist or no
* value is stored at the addressed node
* @throws NullPointerException if {@code key} is {@code null}
*/
public V getNormalizedString(final String key) {
Objects.requireNonNull(key, ARG_KEY);
final CompiledNode<V> node = findNode(key);
if (node == null) {
return null;
}
final V[] orderedValues = node.orderedValues();
if (orderedValues.length == 0) {
return null;
}
return orderedValues[0];
}
/**
* Returns all values stored at the node addressed by the supplied key, ordered
* by descending frequency.
@@ -745,6 +827,7 @@ public final class FrequencyTrie<V> {
*/
private static <V> void writeNode(final DataOutputStream dataOutput, final ValueStreamCodec<V> valueCodec,
final CompiledNode<V> node, final Map<CompiledNode<V>, Integer> nodeIds) throws IOException {
dataOutput.writeBoolean(node.acceptsRemainingInput());
dataOutput.writeInt(node.edgeLabels().length);
for (int index = 0; index < node.edgeLabels().length; index++) {
dataOutput.writeChar(node.edgeLabels()[index]);
@@ -777,6 +860,7 @@ public final class FrequencyTrie<V> {
final V[] values = node.orderedValues();
final int[] counts = node.orderedCounts();
updateInt(messageDigest, node.acceptsRemainingInput() ? 1 : 0);
updateInt(messageDigest, edgeLabels.length);
for (char edgeLabel : edgeLabels) {
updateInt(messageDigest, edgeLabel);
@@ -863,7 +947,7 @@ public final class FrequencyTrie<V> {
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount,
effectiveMaxExpandedIndex);
effectiveMaxExpandedIndex, version);
final CompiledNode<V> rootNode = nodes[rootNodeId];
if (LOGGER.isLoggable(Level.FINE)) {
@@ -880,8 +964,8 @@ public final class FrequencyTrie<V> {
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version)
throws IOException {
if (version == STREAM_VERSION) {
return readTextMetadata(dataInput);
if (version >= TEXT_METADATA_VERSION) {
return readTextMetadata(dataInput, version);
}
final WordTraversalDirection traversalDirection = readTraversalDirection(dataInput, version);
@@ -898,9 +982,10 @@ public final class FrequencyTrie<V> {
caseProcessingMode);
}
private static TrieMetadata readTextMetadata(final DataInputStream dataInput) throws IOException {
private static TrieMetadata readTextMetadata(final DataInputStream dataInput, final int version)
throws IOException {
try {
return TrieMetadata.fromTextBlock(STREAM_VERSION, dataInput.readUTF());
return TrieMetadata.fromTextBlock(version, dataInput.readUTF());
} catch (IllegalArgumentException exception) {
throw new IOException("Invalid metadata block.", exception);
}
@@ -936,14 +1021,19 @@ public final class FrequencyTrie<V> {
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final int nodeCount,
final int maxExpandedIndex) throws IOException {
final int maxExpandedIndex, final int version) throws IOException {
final char[][] edgeLabelsByNode = new char[nodeCount][];
final int[][] childNodeIdsByNode = new int[nodeCount][];
@SuppressWarnings("unchecked")
final V[][] orderedValuesByNode = (V[][]) new Object[nodeCount][];
final int[][] orderedCountsByNode = new int[nodeCount][];
final boolean[] acceptsRemainingInputByNode = new boolean[nodeCount];
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
if (version >= ACCEPTING_NODE_VERSION) {
acceptsRemainingInputByNode[nodeIndex] = dataInput.readBoolean();
}
final int edgeCount = dataInput.readInt();
if (edgeCount < 0) {
throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount);
@@ -963,6 +1053,12 @@ public final class FrequencyTrie<V> {
if (valueCount < 0) {
throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount);
}
if (acceptsRemainingInputByNode[nodeIndex] && edgeCount != 0) {
throw new IOException("Accepting node " + nodeIndex + " cannot have child edges.");
}
if (acceptsRemainingInputByNode[nodeIndex] && valueCount == 0) {
throw new IOException("Accepting node " + nodeIndex + " must store at least one value.");
}
orderedValuesByNode[nodeIndex] = arrayFactory.apply(valueCount);
orderedCountsByNode[nodeIndex] = new int[valueCount];
@@ -983,7 +1079,7 @@ public final class FrequencyTrie<V> {
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
nodes[nodeIndex] = resolveNode(nodeIndex, edgeLabelsByNode, childNodeIdsByNode, orderedValuesByNode,
orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
orderedCountsByNode, acceptsRemainingInputByNode, nodes, inProgress, maxExpandedIndex);
}
return nodes;
@@ -991,8 +1087,8 @@ public final class FrequencyTrie<V> {
private static <V> CompiledNode<V> resolveNode(final int nodeIndex, final char[][] edgeLabelsByNode,
final int[][] childNodeIdsByNode, final V[][] orderedValuesByNode, final int[][] orderedCountsByNode,
final CompiledNode<V>[] nodes, final boolean[] inProgress, final int maxExpandedIndex)
throws IOException {
final boolean[] acceptsRemainingInputByNode, final CompiledNode<V>[] nodes,
final boolean[] inProgress, final int maxExpandedIndex) throws IOException {
final CompiledNode<V> cachedNode = nodes[nodeIndex];
if (cachedNode != null) {
return cachedNode;
@@ -1017,11 +1113,12 @@ public final class FrequencyTrie<V> {
+ ": " + childNodeId);
}
children[edgeIndex] = resolveNode(childNodeId, edgeLabelsByNode, childNodeIdsByNode,
orderedValuesByNode, orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
orderedValuesByNode, orderedCountsByNode, acceptsRemainingInputByNode, nodes, inProgress,
maxExpandedIndex);
}
final CompiledNode<V> node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex],
maxExpandedIndex, orderedCountsByNode[nodeIndex]);
acceptsRemainingInputByNode[nodeIndex], maxExpandedIndex, orderedCountsByNode[nodeIndex]);
nodes[nodeIndex] = node;
return node;
} finally {
@@ -1047,7 +1144,30 @@ public final class FrequencyTrie<V> {
* @return compiled node, or {@code null} if the path does not exist
*/
private CompiledNode<V> findNode(final String key) {
return findNode((CharSequence) key);
CompiledNode<V> current = this.root;
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key.charAt(traversalOffset));
if (current == null) {
return null;
}
}
return current;
}
for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key.charAt(traversalOffset));
if (current == null) {
return null;
}
}
return current;
}
/**
@@ -1060,6 +1180,9 @@ public final class FrequencyTrie<V> {
CompiledNode<V> current = this.root;
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key.charAt(traversalOffset));
if (current == null) {
return null;
@@ -1069,6 +1192,9 @@ public final class FrequencyTrie<V> {
}
for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key.charAt(traversalOffset));
if (current == null) {
return null;
@@ -1089,6 +1215,9 @@ public final class FrequencyTrie<V> {
CompiledNode<V> current = this.root;
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
for (int traversalOffset = offset + length - 1; traversalOffset >= offset; traversalOffset--) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key[traversalOffset]);
if (current == null) {
return null;
@@ -1099,6 +1228,9 @@ public final class FrequencyTrie<V> {
final int endExclusive = offset + length;
for (int traversalOffset = offset; traversalOffset < endExclusive; traversalOffset++) {
if (current.acceptsRemainingInput()) {
return current;
}
current = current.findChild(key[traversalOffset]);
if (current == null) {
return null;
@@ -1535,21 +1667,31 @@ public final class FrequencyTrie<V> {
* @return canonical reduced node
*/
private ReducedNode<V> reduce(final MutableNode<V> source, final ReductionContext<V> context) {
final Map<Character, ReducedNode<V>> reducedChildren = new LinkedHashMap<>();
Map<Character, ReducedNode<V>> reducedChildren = new LinkedHashMap<>();
for (Map.Entry<Character, MutableNode<V>> childEntry : source.children().entrySet()) {
final ReducedNode<V> reducedChild = reduce(childEntry.getValue(), context);
reducedChildren.put(childEntry.getKey(), reducedChild);
}
final Map<V, Integer> localCounts = copyCounts(source.valueCounts());
Map<V, Integer> localCounts = copyCounts(source.valueCounts());
boolean acceptsRemainingInput = false;
if (context.settings().contractUniformSubtrees()) {
final Map<V, Integer> contractedCounts = contractUniformSubtree(localCounts, reducedChildren);
if (!contractedCounts.isEmpty()) {
localCounts = contractedCounts;
reducedChildren = Collections.emptyMap();
acceptsRemainingInput = true;
}
}
final LocalValueSummary<V> localSummary = LocalValueSummary.of(localCounts, this.arrayFactory);
final ReductionSignature<V> signature = ReductionSignature.create(localSummary, reducedChildren,
context.settings());
context.settings(), acceptsRemainingInput);
ReducedNode<V> canonical = context.lookup(signature);
if (canonical == null) {
canonical = new ReducedNode<>(signature, localCounts, reducedChildren);
canonical = new ReducedNode<>(signature, localCounts, reducedChildren, acceptsRemainingInput);
context.register(signature, canonical);
return canonical;
}
@@ -1560,6 +1702,64 @@ public final class FrequencyTrie<V> {
return canonical;
}
/**
* Returns aggregated local counts when the supplied internal subtree contains
* one uniform value, otherwise {@code null}.
*
* @param localCounts local counts at the current node
* @param reducedChildren already reduced children
* @return single-value aggregate for a uniform non-leaf subtree, otherwise an
* empty map
*/
private Map<V, Integer> contractUniformSubtree(final Map<V, Integer> localCounts,
final Map<Character, ReducedNode<V>> reducedChildren) {
if (reducedChildren.isEmpty()) {
return Collections.emptyMap();
}
V uniformValue = null;
boolean valueSeen = false;
if (!localCounts.isEmpty()) {
if (localCounts.size() != SINGLE_VALUE_COUNT) {
return Collections.emptyMap();
}
final Map.Entry<V, Integer> localEntry = localCounts.entrySet().iterator().next();
uniformValue = localEntry.getKey();
valueSeen = true;
}
for (ReducedNode<V> child : reducedChildren.values()) {
if (!isSingleValueLeaf(child)) {
return Collections.emptyMap();
}
final Map.Entry<V, Integer> childEntry = child.localCounts().entrySet().iterator().next();
if (valueSeen && !Objects.equals(uniformValue, childEntry.getKey())) {
return Collections.emptyMap();
}
uniformValue = childEntry.getKey();
valueSeen = true;
}
if (!valueSeen) {
return Collections.emptyMap();
}
final Map<V, Integer> contractedCounts = new LinkedHashMap<>(SINGLE_VALUE_COUNT);
contractedCounts.put(uniformValue, SINGLE_VALUE_COUNT);
return contractedCounts;
}
/**
* Returns whether the reduced node is a leaf with exactly one stored value.
*
* @param node node to inspect
* @return {@code true} when the node can participate in uniform contraction
*/
private boolean isSingleValueLeaf(final ReducedNode<V> node) {
return node.children().isEmpty() && node.localCounts().size() == SINGLE_VALUE_COUNT;
}
/**
* Freezes a reduced node into an immutable compiled node.
*
@@ -1592,7 +1792,7 @@ public final class FrequencyTrie<V> {
}
final CompiledNode<V> frozen = new CompiledNode<>(edges, childNodes, localSummary.orderedValues(),
this.maxExpandedIndex, localSummary.orderedCounts());
reducedNode.acceptsRemainingInput(), this.maxExpandedIndex, localSummary.orderedCounts());
cache.put(reducedNode, frozen);
return frozen;
}

99
src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java
View File

@@ -30,7 +30,10 @@
******************************************************************************/
package org.egothor.stemmer;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.IntFunction;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -115,6 +118,63 @@ public final class FrequencyTrieBuilders {
return copyOf(source, arrayFactory, ReductionSettings.withDefaults(reductionMode));
}
/**
* Reconstructs a compiled trie with every stored value transformed to another
* value type.
*
* <p>
* The method preserves logical keys, local value counts, trie metadata, and the
* supplied reduction settings. It is intended for runtime specialization, such
* as replacing serialized patch-command strings with precompiled patch command
* objects without changing the persisted binary trie format.
* </p>
*
* @param source source compiled trie
* @param arrayFactory array factory for mapped values
* @param reductionSettings reduction settings for the mapped trie
* @param valueMapper value mapping function
* @param <S> source value type
* @param <T> target value type
* @return compiled trie containing mapped values
* @throws NullPointerException if any argument is {@code null}
*/
public static <S, T> FrequencyTrie<T> mapValues(final FrequencyTrie<S> source,
final IntFunction<T[]> arrayFactory, final ReductionSettings reductionSettings,
final Function<? super S, ? extends T> valueMapper) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(arrayFactory, "arrayFactory");
Objects.requireNonNull(reductionSettings, "reductionSettings");
Objects.requireNonNull(valueMapper, "valueMapper");
final Map<CompiledNode<S>, CompiledNode<T>> cache = new IdentityHashMap<>();
final CompiledNode<T> mappedRoot = mapCompiledNode(source.root(), arrayFactory, valueMapper, cache);
final TrieMetadata metadata = TrieMetadata.forCompilation(source.traversalDirection(), reductionSettings,
source.metadata().diacriticProcessingMode(), source.metadata().caseProcessingMode());
LOGGER.log(Level.FINE, "Mapped compiled trie values to a specialized value type.");
return FrequencyTrie.fromCompiled(arrayFactory, mappedRoot, metadata);
}
/**
* Reconstructs a compiled trie with every stored value transformed to another
* value type using default settings for the supplied reduction mode.
*
* @param source source compiled trie
* @param arrayFactory array factory for mapped values
* @param reductionMode reduction mode for the mapped trie
* @param valueMapper value mapping function
* @param <S> source value type
* @param <T> target value type
* @return compiled trie containing mapped values
* @throws NullPointerException if any argument is {@code null}
*/
public static <S, T> FrequencyTrie<T> mapValues(final FrequencyTrie<S> source,
final IntFunction<T[]> arrayFactory, final ReductionMode reductionMode,
final Function<? super S, ? extends T> valueMapper) {
Objects.requireNonNull(reductionMode, "reductionMode");
return mapValues(source, arrayFactory, ReductionSettings.withDefaults(reductionMode), valueMapper);
}
/**
* Copies one compiled node and all reachable descendants into the target
* builder.
@@ -138,4 +198,43 @@ public final class FrequencyTrieBuilders {
keyBuilder.setLength(keyBuilder.length() - 1);
}
}
/**
* Maps one compiled node graph while preserving canonical sharing and accepting
* leaf semantics.
*
* @param node source node
* @param arrayFactory target value array factory
* @param valueMapper value mapper
* @param cache identity cache for shared compiled nodes
* @param <S> source value type
* @param <T> target value type
* @return mapped compiled node
*/
private static <S, T> CompiledNode<T> mapCompiledNode(final CompiledNode<S> node,
final IntFunction<T[]> arrayFactory, final Function<? super S, ? extends T> valueMapper,
final Map<CompiledNode<S>, CompiledNode<T>> cache) {
final CompiledNode<T> existing = cache.get(node);
if (existing != null) {
return existing;
}
final CompiledNode<S>[] sourceChildren = node.children();
@SuppressWarnings("unchecked")
final CompiledNode<T>[] mappedChildren = new CompiledNode[sourceChildren.length];
for (int childIndex = 0; childIndex < sourceChildren.length; childIndex++) {
mappedChildren[childIndex] = mapCompiledNode(sourceChildren[childIndex], arrayFactory, valueMapper, cache);
}
final S[] sourceValues = node.orderedValues();
final T[] mappedValues = arrayFactory.apply(sourceValues.length);
for (int valueIndex = 0; valueIndex < sourceValues.length; valueIndex++) {
mappedValues[valueIndex] = valueMapper.apply(sourceValues[valueIndex]);
}
final CompiledNode<T> mapped = new CompiledNode<>(node.edgeLabels().clone(), mappedChildren, mappedValues,
node.acceptsRemainingInput(), CompiledNode.DEFAULT_MAX_EXPANDED_INDEX, node.orderedCounts().clone());
cache.put(node, mapped);
return mapped;
}
}

202
src/main/java/org/egothor/stemmer/PatchCommandEncoder.java
View File

@@ -70,16 +70,6 @@ import java.util.concurrent.locks.ReentrantLock;
@SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" })
public final class PatchCommandEncoder {
/**
* Backward direction apply strategy with no runtime direction branching.
*/
private static final ApplyStrategy BACKWARD_APPLY_STRATEGY = PatchCommandEncoder::applyBackward;
/**
* Forward direction apply strategy with no runtime direction branching.
*/
private static final ApplyStrategy FORWARD_APPLY_STRATEGY = PatchCommandEncoder::applyForward;
/**
* Serialized opcode for deleting one or more characters.
*/
@@ -175,9 +165,9 @@ public final class PatchCommandEncoder {
private final WordTraversalDirection traversalDirection;
/**
* Direction-specialized patch apply strategy.
* Whether this instance applies patch commands in backward traversal order.
*/
private final ApplyStrategy applyStrategy;
private final boolean backwardTraversal;
/**
* Currently allocated source dimension of reusable matrices.
@@ -222,21 +212,6 @@ public final class PatchCommandEncoder {
MATCH
}
/**
* Direction-specialized patch application strategy.
*/
@FunctionalInterface
private interface ApplyStrategy {
/**
* Applies the command.
*
* @param source original text
* @param patchCommand patch command
* @return final text after applying the command
*/
String apply(String source, String patchCommand);
}
private PatchCommandEncoder(final Builder builder) {
this.traversalDirection = Objects.requireNonNull(builder.traversalDirection, "traversalDirection");
final int insertCost = builder.insertCost;
@@ -260,7 +235,7 @@ public final class PatchCommandEncoder {
this.deleteCost = deleteCost;
this.replaceCost = replaceCost;
this.matchCost = matchCost;
this.applyStrategy = applyStrategyFor(this.traversalDirection);
this.backwardTraversal = this.traversalDirection == WordTraversalDirection.BACKWARD;
this.sourceCapacity = 0;
this.targetCapacity = 0;
this.costMatrix = new int[0][0];
@@ -304,19 +279,43 @@ public final class PatchCommandEncoder {
* direction.
*
* <p>
* This is the branch-free instance-level fast path for repeated patch
* application in a known traversal direction.
* This is the instance-level fast path for repeated patch application in a
* known traversal direction. It avoids the static API null and direction
* validation path and calls the selected decoder directly.
* </p>
*
* @param source original source word
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
* @deprecated Since 2.3.0. Runtime stemming should compile
* {@code patchCommand} once through {@link #compile(String)} and
* reuse {@link CompiledPatchCommand#apply(String)}. The
* String-based application path reparses the patch command on every
* call and is kept only for source compatibility before the 3.0.0
* migration.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public String applyWithConfiguredDirection(final String source, final String patchCommand) {
if (source == null) {
return null;
}
return this.applyStrategy.apply(source, patchCommand);
if (this.backwardTraversal) {
return applyBackwardNonNull(source, patchCommand);
}
return applyForwardNonNull(source, patchCommand);
}
/**
* Compiles a patch command for repeated application with this encoder
* instance traversal direction.
*
* @param patchCommand compact patch command
* @return immutable compiled patch command
* @throws IllegalArgumentException if the serialized command contains an
* unsupported opcode or invalid NOOP argument
*/
public CompiledPatchCommand compile(final String patchCommand) {
return CompiledPatchCommand.compile(patchCommand, this.traversalDirection);
}
/**
@@ -326,7 +325,14 @@ public final class PatchCommandEncoder {
* @param source original source word
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
* @deprecated Since 2.3.0. Runtime stemming should use
* {@link CompiledPatchCommand#compile(String, WordTraversalDirection)}
* once and then reuse {@link CompiledPatchCommand#apply(String)}.
* This method repeatedly interprets the serialized patch-command
* string and is retained only for compatibility before the 3.0.0
* migration.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static String apply(final String source, final String patchCommand) {
return apply(source, patchCommand, WordTraversalDirection.BACKWARD);
}
@@ -343,14 +349,41 @@ public final class PatchCommandEncoder {
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
* @deprecated Since 2.3.0. Runtime stemming should use
* {@link CompiledPatchCommand#compile(String, WordTraversalDirection)}
* once and then reuse {@link CompiledPatchCommand#apply(String)}.
* This method repeatedly interprets the serialized patch-command
* string and is retained only for compatibility before the 3.0.0
* migration.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static String apply(final String source, final String patchCommand,
final WordTraversalDirection traversalDirection) {
Objects.requireNonNull(traversalDirection, "traversalDirection");
if (source == null) {
return null;
}
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
if (traversalDirection == WordTraversalDirection.BACKWARD) {
return applyBackwardNonNull(source, patchCommand);
}
return applyForwardNonNull(source, patchCommand);
}
/**
* Compiles a patch command for repeated application with the supplied
* traversal direction.
*
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @return immutable compiled patch command
* @throws NullPointerException if {@code traversalDirection} is
* {@code null}
* @throws IllegalArgumentException if the serialized command contains an
* unsupported opcode or invalid NOOP argument
*/
public static CompiledPatchCommand compile(final String patchCommand,
final WordTraversalDirection traversalDirection) {
return CompiledPatchCommand.compile(patchCommand, traversalDirection);
}
/**
@@ -371,7 +404,15 @@ public final class PatchCommandEncoder {
* @param outputLength writable output capacity
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
* when {@code outputLength} is too small
* @deprecated Since 2.3.0. Compile {@code patchCommand} once through
* {@link #compile(String, WordTraversalDirection)} and call
* {@link CompiledPatchCommand#applyTo(CharSequence, char[], int, int)}
* or
* {@link CompiledPatchCommand#applyTo(CharSequence, int, int, char[], int, int)}.
* This String-based method reparses patch commands on every call and
* is kept only for compatibility before the 3.0.0 migration.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static int applyTo(final CharSequence source, final String patchCommand,
final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset,
final int outputLength) {
@@ -405,7 +446,13 @@ public final class PatchCommandEncoder {
* when {@code outputLength} is too small
* @throws IllegalArgumentException when source and output ranges overlap in the
* same array
* @deprecated Since 2.3.0. Compile {@code patchCommand} once through
* {@link #compile(String, WordTraversalDirection)} and call
* {@link CompiledPatchCommand#applyTo(char[], int, int, char[], int, int)}.
* This String-based method reparses patch commands on every call and
* is kept only for compatibility before the 3.0.0 migration.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int outputLength) {
@@ -484,36 +531,33 @@ public final class PatchCommandEncoder {
/**
* Applies a patch command using the historical backward Egothor semantics.
*
* @param source original source word in legacy backward logical space
* @param source non-null original source word in legacy backward logical
* space
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
* @return transformed word
*/
private static String applyBackward(final String source, final String patchCommand) {
if (source == null) {
return null;
}
if (patchCommand == null || patchCommand.isEmpty()) {
private static String applyBackwardNonNull(final String source, final String patchCommand) {
if (patchCommand == null) {
return source;
}
if (NOOP_PATCH.equals(patchCommand)) {
final int patchLength = patchCommand.length();
if (patchLength == 0 || (patchLength & 1) != 0) {
return source;
}
if ((patchCommand.length() & 1) != 0) {
return source;
}
if (patchCommand.length() == 2) {
if (patchLength == 2) {
return applySingleBackwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1));
}
final StringBuilder result = new StringBuilder(source);
if (result.isEmpty()) {
return applyBackwardToEmptySource(result, patchCommand);
if (source.isEmpty()) {
return applyBackwardToEmptySource(patchCommand);
}
final StringBuilder result = new StringBuilder(source);
int position = result.length() - 1;
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -547,12 +591,12 @@ public final class PatchCommandEncoder {
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return source;
default:
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position--;
@@ -567,36 +611,32 @@ public final class PatchCommandEncoder {
/**
* Applies a patch command using forward traversal semantics.
*
* @param source original source word
* @param source non-null original source word
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
* @return transformed word
*/
private static String applyForward(final String source, final String patchCommand) {
if (source == null) {
return null;
}
if (patchCommand == null || patchCommand.isEmpty()) {
private static String applyForwardNonNull(final String source, final String patchCommand) {
if (patchCommand == null) {
return source;
}
if (NOOP_PATCH.equals(patchCommand)) {
final int patchLength = patchCommand.length();
if (patchLength == 0 || (patchLength & 1) != 0) {
return source;
}
if ((patchCommand.length() & 1) != 0) {
return source;
}
if (patchCommand.length() == 2) {
if (patchLength == 2) {
return applySingleForwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1));
}
final StringBuilder result = new StringBuilder(source);
if (result.isEmpty()) {
return applyForwardToEmptySource(result, patchCommand);
if (source.isEmpty()) {
return applyForwardToEmptySource(patchCommand);
}
final StringBuilder result = new StringBuilder(source);
int position = 0;
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
for (int patchIndex = 0; patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -628,12 +668,12 @@ public final class PatchCommandEncoder {
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return source;
default:
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position++;
@@ -751,12 +791,12 @@ public final class PatchCommandEncoder {
* behavior for index-invalid commands.
* </p>
*
* @param result empty result builder
* @param patchCommand compact patch command
* @return transformed word, or the original empty word when the patch is
* malformed
*/
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
private static String applyBackwardToEmptySource(final String patchCommand) {
final StringBuilder result = new StringBuilder(patchCommand.length() >> 1);
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
@@ -774,12 +814,12 @@ public final class PatchCommandEncoder {
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return "";
default:
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
} catch (IndexOutOfBoundsException exception) {
@@ -792,12 +832,12 @@ public final class PatchCommandEncoder {
/**
* Applies a forward patch command to an empty source word.
*
* @param result empty result builder
* @param patchCommand compact patch command
* @return transformed word, or the original empty word when the patch is
* malformed
*/
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
private static String applyForwardToEmptySource(final String patchCommand) {
final StringBuilder result = new StringBuilder(patchCommand.length() >> 1);
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
@@ -815,12 +855,12 @@ public final class PatchCommandEncoder {
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return "";
default:
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
} catch (IndexOutOfBoundsException exception) {
@@ -1535,16 +1575,6 @@ public final class PatchCommandEncoder {
}
}
/**
* Returns the direction-specialized apply strategy.
*
* @param traversalDirection requested traversal direction
* @return branch-free apply strategy for that direction
*/
private static ApplyStrategy applyStrategyFor(final WordTraversalDirection traversalDirection) {
return traversalDirection == WordTraversalDirection.BACKWARD ? BACKWARD_APPLY_STRATEGY : FORWARD_APPLY_STRATEGY;
}
/**
* Decodes a compact count argument used by skip and delete instructions.
*

41
src/main/java/org/egothor/stemmer/ReductionSettings.java
View File

@@ -42,10 +42,13 @@ import java.util.Objects;
* @param reductionMode reduction mode
* @param dominantWinnerMinPercent minimum dominant winner percentage
* @param dominantWinnerOverSecondRatio minimum winner-over-second ratio
* @param contractUniformSubtrees whether compilation may contract a subtree
* whose reachable terminal values all contain
* the same single value
*/
@SuppressWarnings("PMD.LongVariable")
public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerMinPercent,
int dominantWinnerOverSecondRatio) {
int dominantWinnerOverSecondRatio, boolean contractUniformSubtrees) {
/**
* Default minimum dominant winner percentage.
@@ -65,12 +68,14 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM
* the inclusive range {@code 1..100}
* @param dominantWinnerOverSecondRatio minimum winner-over-second ratio, must
* be at least {@code 1}
* @param contractUniformSubtrees whether uniform subtrees may be
* contracted into accepting leaves
* @throws NullPointerException if {@code reductionMode} is {@code null}
* @throws IllegalArgumentException if any numeric value is outside the valid
* range
*/
public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent,
final int dominantWinnerOverSecondRatio) {
final int dominantWinnerOverSecondRatio, final boolean contractUniformSubtrees) {
this.reductionMode = Objects.requireNonNull(reductionMode, "reductionMode");
if (dominantWinnerMinPercent < 1 || dominantWinnerMinPercent > 100) {
throw new IllegalArgumentException("dominantWinnerMinPercent must be in range 1..100.");
@@ -80,6 +85,19 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM
}
this.dominantWinnerMinPercent = dominantWinnerMinPercent;
this.dominantWinnerOverSecondRatio = dominantWinnerOverSecondRatio;
this.contractUniformSubtrees = contractUniformSubtrees;
}
/**
* Creates a new instance without uniform-subtree contraction.
*
* @param reductionMode reduction mode
* @param dominantWinnerMinPercent minimum dominant winner percentage
* @param dominantWinnerOverSecondRatio minimum winner-over-second ratio
*/
public ReductionSettings(final ReductionMode reductionMode, final int dominantWinnerMinPercent,
final int dominantWinnerOverSecondRatio) {
this(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio, false);
}
/**
@@ -93,4 +111,23 @@ public record ReductionSettings(ReductionMode reductionMode, int dominantWinnerM
return new ReductionSettings(reductionMode, DEFAULT_DOMINANT_WINNER_MIN_PERCENT,
DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO);
}
/**
* Returns settings that run uniform-subtree contraction before the configured
* subtree-merging mode.
*
* <p>
* This is intended for Radixor patch-command tries, where a contracted accepting
* leaf can safely represent a subtree whose reachable entries all use the same
* patch command.
* </p>
*
* @param settings base settings
* @return equivalent settings with uniform-subtree contraction enabled
*/
/* default */ static ReductionSettings withUniformSubtreeContraction(final ReductionSettings settings) {
Objects.requireNonNull(settings, "settings");
return new ReductionSettings(settings.reductionMode(), settings.dominantWinnerMinPercent(),
settings.dominantWinnerOverSecondRatio(), true);
}
}

1
src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java
View File

@@ -349,6 +349,7 @@ public final class StemmerKnowledgeExperiment {
* @param trie compiled trie under test
* @return immutable counts for this single input
*/
@SuppressWarnings("deprecation")
private static EvaluationCounts evaluateInput(final String input, final String expectedStem,
final FrequencyTrie<String> trie) {
long getCorrect = 0L;

506
src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java
View File

@@ -39,6 +39,8 @@ import java.io.PushbackInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -68,6 +70,7 @@ import java.util.zip.GZIPInputStream;
* items containing Unicode whitespace characters while reporting them through
* aggregated warning log records.
*/
@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyMethods" })
public final class StemmerPatchTrieLoader {
/* default */ static final String FILENAME_REQUIRED = "fileName required";
@@ -293,7 +296,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Language, boolean, ReductionSettings)} so
* patch commands are represented as {@link CompiledPatchCommand}
* values instead of reparsed {@link String} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Language language, final boolean storeOriginal,
final ReductionSettings reductionSettings) throws IOException {
Objects.requireNonNull(language, "language");
@@ -303,6 +311,30 @@ public final class StemmerPatchTrieLoader {
return load(language, storeOriginal, metadata);
}
/**
* Loads a bundled dictionary and returns a runtime-specialized trie whose
* values are compiled patch commands.
*
* <p>
* The text dictionary is still compiled through the canonical serialized
* patch-command representation. The returned trie replaces each stored
* serialized patch command with a {@link CompiledPatchCommand} so repeated
* runtime stemming does not parse patch-command strings.
* </p>
*
* @param language bundled language dictionary
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionSettings reduction settings
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Language language,
final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException {
return compilePatchTrie(load(language, storeOriginal, reductionSettings));
}
/**
* Loads a bundled dictionary using explicit trie compilation metadata.
*
@@ -320,7 +352,11 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Language, boolean, TrieMetadata)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Language language, final boolean storeOriginal,
final TrieMetadata metadata) throws IOException {
Objects.requireNonNull(language, "language");
@@ -335,6 +371,23 @@ public final class StemmerPatchTrieLoader {
}
}
/**
* Loads a bundled dictionary using explicit trie compilation metadata and
* returns a runtime-specialized trie whose values are compiled patch commands.
*
* @param language bundled language dictionary
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param metadata trie metadata describing the compilation configuration
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Language language,
final boolean storeOriginal, final TrieMetadata metadata) throws IOException {
return compilePatchTrie(load(language, storeOriginal, metadata));
}
/**
* Loads a bundled dictionary using default settings for the supplied reduction
* mode.
@@ -354,13 +407,35 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Language, boolean, ReductionMode)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Language language, final boolean storeOriginal,
final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(reductionMode, "reductionMode");
return load(language, storeOriginal, ReductionSettings.withDefaults(reductionMode));
}
/**
* Loads a bundled dictionary using default settings for the supplied reduction
* mode and returns a runtime-specialized trie whose values are compiled patch
* commands.
*
* @param language bundled language dictionary
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the dictionary cannot be found or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Language language,
final boolean storeOriginal, final ReductionMode reductionMode) throws IOException {
return compilePatchTrie(load(language, storeOriginal, reductionMode));
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings.
*
@@ -379,13 +454,35 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, ReductionSettings)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionSettings reductionSettings) throws IOException {
return load(path, storeOriginal, reductionSettings, WordTraversalDirection.BACKWARD,
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings
* and returns a runtime-specialized trie whose values are compiled patch
* commands.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionSettings reduction settings
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException {
return compilePatchTrie(load(path, storeOriginal, reductionSettings));
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings
* and explicit traversal direction.
@@ -405,7 +502,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection)
throws IOException {
@@ -413,6 +515,26 @@ public final class StemmerPatchTrieLoader {
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings
* and traversal direction, returning runtime-specialized compiled patch values.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted using
* the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys and
* patch commands
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection) throws IOException {
return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection));
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings,
* explicit traversal direction, and explicit case processing mode.
@@ -432,7 +554,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode) throws IOException {
@@ -440,6 +567,29 @@ public final class StemmerPatchTrieLoader {
DiacriticProcessingMode.AS_IS);
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings,
* traversal direction, and case processing mode, returning runtime-specialized
* compiled patch values.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted using
* the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys and
* patch commands
* @param caseProcessingMode case processing mode used during dictionary parsing
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode)
throws IOException {
return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode));
}
/**
* Loads a dictionary from a filesystem path using explicit reduction settings,
* traversal direction, case processing mode, and diacritic processing mode.
@@ -457,7 +607,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode, DiacriticProcessingMode)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
@@ -468,6 +623,32 @@ public final class StemmerPatchTrieLoader {
return load(path, storeOriginal, metadata);
}
/**
* Loads a dictionary from a filesystem path using explicit semantic metadata
* dimensions, returning runtime-specialized compiled patch values.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted
* using the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys
* and patch commands
* @param caseProcessingMode case processing mode used during dictionary
* parsing
* @param diacriticProcessingMode diacritic processing mode used during
* dictionary parsing
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
final DiacriticProcessingMode diacriticProcessingMode) throws IOException {
return compilePatchTrie(load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
diacriticProcessingMode));
}
/**
* Loads a dictionary from a filesystem path using explicit trie compilation
* metadata.
@@ -485,7 +666,11 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, TrieMetadata)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
throws IOException {
Objects.requireNonNull(path, PARAMETER_PATH);
@@ -498,6 +683,24 @@ public final class StemmerPatchTrieLoader {
}
}
/**
* Loads a dictionary from a filesystem path using explicit trie compilation
* metadata and returns a runtime-specialized trie whose values are compiled
* patch commands.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param metadata trie metadata describing the compilation configuration
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final TrieMetadata metadata) throws IOException {
return compilePatchTrie(load(path, storeOriginal, metadata));
}
/**
* Loads a dictionary from a filesystem path using default settings for the
* supplied reduction mode.
@@ -518,13 +721,35 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(Path, boolean, ReductionMode)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(reductionMode, "reductionMode");
return load(path, storeOriginal, ReductionSettings.withDefaults(reductionMode));
}
/**
* Loads a dictionary from a filesystem path using default settings for the
* supplied reduction mode and returns runtime-specialized compiled patch
* values.
*
* @param path path to the dictionary file
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final Path path,
final boolean storeOriginal, final ReductionMode reductionMode) throws IOException {
return compilePatchTrie(load(path, storeOriginal, reductionMode));
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings.
@@ -543,13 +768,36 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionSettings)} so
* patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final ReductionSettings reductionSettings) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return load(Path.of(fileName), storeOriginal, reductionSettings);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings and returns runtime-specialized compiled patch values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionSettings reduction settings
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings and explicit traversal direction.
@@ -571,7 +819,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection)
throws IOException {
@@ -580,6 +833,28 @@ public final class StemmerPatchTrieLoader {
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings and traversal direction, returning runtime-specialized compiled
* patch values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted using
* the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys and
* patch commands
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings, explicit traversal direction, and explicit case processing mode.
@@ -600,7 +875,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode) throws IOException {
@@ -609,6 +889,31 @@ public final class StemmerPatchTrieLoader {
DiacriticProcessingMode.AS_IS);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings, traversal direction, and case processing mode, returning
* runtime-specialized compiled patch values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted using
* the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys and
* patch commands
* @param caseProcessingMode case processing mode used during dictionary parsing
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode)
throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection,
caseProcessingMode);
}
/**
* Loads a dictionary from a filesystem path string using explicit reduction
* settings, explicit traversal direction, explicit case processing mode, and
@@ -627,7 +932,12 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode, DiacriticProcessingMode)}
* so patch commands are represented as
* {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
@@ -637,6 +947,33 @@ public final class StemmerPatchTrieLoader {
diacriticProcessingMode);
}
/**
* Loads a dictionary from a filesystem path string using explicit semantic
* metadata dimensions, returning runtime-specialized compiled patch values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted
* using the canonical no-op patch command
* @param reductionSettings reduction settings
* @param traversalDirection traversal direction used for both trie keys
* and patch commands
* @param caseProcessingMode case processing mode used during dictionary
* parsing
* @param diacriticProcessingMode diacritic processing mode used during
* dictionary parsing
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final ReductionSettings reductionSettings,
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
final DiacriticProcessingMode diacriticProcessingMode) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection,
caseProcessingMode, diacriticProcessingMode);
}
/**
* Loads a dictionary from a filesystem path string using explicit trie
* compilation metadata.
@@ -652,13 +989,35 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, TrieMetadata)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final TrieMetadata metadata) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return load(Path.of(fileName), storeOriginal, metadata);
}
/**
* Loads a dictionary from a filesystem path string using explicit trie
* compilation metadata and returns runtime-specialized compiled patch values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param metadata trie metadata describing the compilation configuration
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final TrieMetadata metadata) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, metadata);
}
/**
* Loads a dictionary from a filesystem path string using default settings for
* the supplied reduction mode.
@@ -677,13 +1036,36 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionMode)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return load(Path.of(fileName), storeOriginal, reductionMode);
}
/**
* Loads a dictionary from a filesystem path string using default settings for
* the supplied reduction mode and returns runtime-specialized compiled patch
* values.
*
* @param fileName file name or path string
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
final boolean storeOriginal, final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionMode);
}
/**
* Parses one dictionary and builds the compiled trie.
*
@@ -736,7 +1118,9 @@ public final class StemmerPatchTrieLoader {
Objects.requireNonNull(reductionSettings, "reductionSettings");
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
return TrieMetadata.forCompilation(traversalDirection, reductionSettings, diacriticProcessingMode,
final ReductionSettings patchReductionSettings = ReductionSettings
.withUniformSubtreeContraction(reductionSettings);
return TrieMetadata.forCompilation(traversalDirection, patchReductionSettings, diacriticProcessingMode,
caseProcessingMode);
}
@@ -750,6 +1134,28 @@ public final class StemmerPatchTrieLoader {
return language.isRightToLeft() ? WordTraversalDirection.FORWARD : WordTraversalDirection.BACKWARD;
}
/**
* Maps textual patch commands to runtime-specialized compiled patch commands.
*
* <p>
* Equal textual patch commands are compiled once and shared by all trie values
* that reference them. The returned trie preserves the source trie keys,
* metadata, traversal direction, counts, and reduction settings.
* </p>
*
* @param trie source trie containing textual patch commands
* @return equivalent trie containing compiled patch commands
* @throws NullPointerException if {@code trie} is {@code null}
*/
private static FrequencyTrie<CompiledPatchCommand> compilePatchTrie(final FrequencyTrie<String> trie) {
final FrequencyTrie<String> sourceTrie = Objects.requireNonNull(trie, "trie");
final Map<String, CompiledPatchCommand> compiledPatches = new HashMap<>(4096);
return FrequencyTrieBuilders.mapValues(sourceTrie, CompiledPatchCommand[]::new,
sourceTrie.metadata().reductionSettings(),
patch -> compiledPatches.computeIfAbsent(patch,
value -> CompiledPatchCommand.compile(value, sourceTrie.traversalDirection())));
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path.
*
@@ -758,12 +1164,30 @@ public final class StemmerPatchTrieLoader {
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadBinaryCompiled(Path)} so patch commands are
* represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> loadBinary(final Path path) throws IOException {
Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.read(path);
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path and
* returns runtime-specialized compiled patch values.
*
* @param path path to the compressed binary trie file
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path) throws IOException {
return compilePatchTrie(loadBinary(path));
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* using a custom dense lookup span override.
@@ -779,12 +1203,34 @@ public final class StemmerPatchTrieLoader {
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadBinaryCompiled(Path, int)} so patch commands are
* represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> loadBinary(final Path path, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex);
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path using
* a custom dense lookup span override and returns runtime-specialized compiled
* patch values.
*
* @param path path to the compressed binary trie file
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path,
final int maxExpandedIndex) throws IOException {
return compilePatchTrie(loadBinary(path, maxExpandedIndex));
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* string.
@@ -794,12 +1240,30 @@ public final class StemmerPatchTrieLoader {
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadBinaryCompiled(String)} so patch commands are
* represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> loadBinary(final String fileName) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.read(fileName);
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* string and returns runtime-specialized compiled patch values.
*
* @param fileName file name or path string
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName) throws IOException {
return compilePatchTrie(loadBinary(fileName));
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* string using a custom dense lookup span override.
@@ -815,13 +1279,35 @@ public final class StemmerPatchTrieLoader {
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadBinaryCompiled(String, int)} so patch commands are
* represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex);
}
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path string
* using a custom dense lookup span override and returns runtime-specialized
* compiled patch values.
*
* @param fileName file name or path string
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName,
final int maxExpandedIndex) throws IOException {
return compilePatchTrie(loadBinary(fileName, maxExpandedIndex));
}
/**
* Loads a GZip-compressed binary patch-command trie from an input stream.
*
@@ -829,12 +1315,30 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if the stream cannot be decompressed or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadBinaryCompiled(InputStream)} so patch commands are
* represented as {@link CompiledPatchCommand} values.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> loadBinary(final InputStream inputStream) throws IOException {
Objects.requireNonNull(inputStream, "inputStream");
return StemmerPatchTrieBinaryIO.read(inputStream);
}
/**
* Loads a GZip-compressed binary patch-command trie from an input stream and
* returns runtime-specialized compiled patch values.
*
* @param inputStream source input stream
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if the stream cannot be decompressed or read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final InputStream inputStream)
throws IOException {
return compilePatchTrie(loadBinary(inputStream));
}
/**
* Loads only persisted metadata from a GZip-compressed binary patch-command
* trie file.

7
src/main/java/org/egothor/stemmer/TrieMetadata.java
View File

@@ -167,6 +167,8 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi
.append("dominantWinnerOverSecondRatio=").append(this.reductionSettings.dominantWinnerOverSecondRatio())
.append('\n')
//
.append("contractUniformSubtrees=").append(this.reductionSettings.contractUniformSubtrees()).append('\n')
//
.append("diacriticProcessingMode=").append(this.diacriticProcessingMode.name()).append('\n')
//
.append("caseProcessingMode=").append(this.caseProcessingMode.name()).append('\n');
@@ -207,13 +209,16 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi
final int dominantWinnerMinPercent = Integer.parseInt(requireEntry(entries, "dominantWinnerMinPercent"));
final int dominantWinnerOverSecondRatio = Integer // NOPMD
.parseInt(requireEntry(entries, "dominantWinnerOverSecondRatio"));
final boolean contractUniformSubtrees = Boolean
.parseBoolean(entries.getOrDefault("contractUniformSubtrees", "false"));
final DiacriticProcessingMode diacriticProcessingMode = DiacriticProcessingMode
.valueOf(requireEntry(entries, "diacriticProcessingMode"));
final CaseProcessingMode caseProcessingMode = CaseProcessingMode
.valueOf(requireEntry(entries, "caseProcessingMode"));
return new TrieMetadata(formatVersion, traversalDirection,
new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio),
new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio,
contractUniformSubtrees),
diacriticProcessingMode, caseProcessingMode);
}

43
src/main/java/org/egothor/stemmer/trie/CompiledNode.java
View File

@@ -93,6 +93,11 @@ public final class CompiledNode<V> {
*/
private final int[] orderedCounts;
/**
* Whether this node accepts any remaining lookup input.
*/
private final boolean acceptsRemainingInput;
/**
* Creates one validated compiled node using {@link #DEFAULT_MAX_EXPANDED_INDEX}
* for dense lookup sizing.
@@ -120,6 +125,23 @@ public final class CompiledNode<V> {
*/
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
final int maxExpandedIndex, final int... orderedCounts) {
this(edgeLabels, children, orderedValues, false, maxExpandedIndex, orderedCounts);
}
/**
* Creates one validated compiled node.
*
* @param acceptsRemainingInput whether this node accepts any remaining lookup
* input
* @param maxExpandedIndex upper bound for the dense lookup interval size
* @throws NullPointerException if any array argument is {@code null}
* @throws IllegalArgumentException if the edge-related arrays or value-related
* arrays do not have matching lengths, the
* dense interval size is negative, or an
* accepting node has children
*/
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
final boolean acceptsRemainingInput, final int maxExpandedIndex, final int... orderedCounts) {
Objects.requireNonNull(edgeLabels, "edgeLabels");
Objects.requireNonNull(children, "children");
Objects.requireNonNull(orderedValues, "orderedValues");
@@ -135,11 +157,18 @@ public final class CompiledNode<V> {
if (orderedValues.length != orderedCounts.length) {
throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length.");
}
if (acceptsRemainingInput && edgeLabels.length != 0) {
throw new IllegalArgumentException("Accepting nodes cannot have child edges.");
}
if (acceptsRemainingInput && orderedValues.length == 0) {
throw new IllegalArgumentException("Accepting nodes must store at least one value.");
}
this.edgeLabels = edgeLabels;
this.children = children;
this.orderedValues = orderedValues;
this.orderedCounts = orderedCounts;
this.acceptsRemainingInput = acceptsRemainingInput;
if (edgeLabels.length == 0 || maxExpandedIndex == 0) {
this.denseChildren = null;
@@ -268,6 +297,15 @@ public final class CompiledNode<V> {
return !hasChildren();
}
/**
* Indicates whether this node accepts any remaining lookup input.
*
* @return {@code true} for a contracted accepting leaf
*/
public boolean acceptsRemainingInput() {
return this.acceptsRemainingInput;
}
/**
* Tests whether an edge label is present at this node.
*
@@ -310,6 +348,7 @@ public final class CompiledNode<V> {
hash = 31 * hash + Arrays.hashCode(this.orderedValues);
hash = 31 * hash + Arrays.hashCode(this.orderedCounts);
hash = 31 * hash + Objects.hash(this.denseEdgeMin);
hash = 31 * hash + Boolean.hashCode(this.acceptsRemainingInput);
hash = 31 * hash + (hasDenseLookup() ? Arrays.hashCode(this.denseChildren) : 0);
return hash;
}
@@ -331,6 +370,7 @@ public final class CompiledNode<V> {
return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children)
&& Arrays.equals(this.orderedValues, other.orderedValues)
&& Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin
&& this.acceptsRemainingInput == other.acceptsRemainingInput
&& Arrays.equals(this.denseChildren, other.denseChildren);
}
@@ -342,7 +382,8 @@ public final class CompiledNode<V> {
@Override
public String toString() {
return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount="
+ this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}';
+ this.orderedValues.length + ", acceptsRemainingInput=" + this.acceptsRemainingInput
+ ", denseTableLength=" + denseTableLength() + '}';
}
/**

35
src/main/java/org/egothor/stemmer/trie/ReducedNode.java
View File

@@ -60,18 +60,38 @@ public final class ReducedNode<V> {
*/
private final Map<Character, ReducedNode<V>> children;
/**
* Whether this reduced node accepts any remaining lookup input.
*/
private final boolean acceptsRemainingInput;
/**
* Creates a new reduced node.
*
* @param signature reduction signature
* @param localCounts local counts
* @param children children
* @param acceptsRemainingInput whether this node accepts any remaining lookup
* input
*/
public ReducedNode(final ReductionSignature<V> signature, final Map<V, Integer> localCounts,
final Map<Character, ReducedNode<V>> children, final boolean acceptsRemainingInput) {
this.signature = signature;
this.localCounts = new LinkedHashMap<>(localCounts);
this.children = new LinkedHashMap<>(children);
this.acceptsRemainingInput = acceptsRemainingInput;
}
/**
* Creates a new non-accepting reduced node.
*
* @param signature reduction signature
* @param localCounts local counts
* @param children children
*/
public ReducedNode(final ReductionSignature<V> signature, final Map<V, Integer> localCounts,
final Map<Character, ReducedNode<V>> children) {
this.signature = signature;
this.localCounts = new LinkedHashMap<>(localCounts);
this.children = new LinkedHashMap<>(children);
this(signature, localCounts, children, false);
}
/**
@@ -111,6 +131,15 @@ public final class ReducedNode<V> {
return this.children;
}
/**
* Returns whether this node accepts any remaining lookup input.
*
* @return {@code true} for a contracted accepting leaf
*/
public boolean acceptsRemainingInput() {
return this.acceptsRemainingInput;
}
/**
* Merges additional local counts into this node.
*

37
src/main/java/org/egothor/stemmer/trie/ReductionSignature.java
View File

@@ -55,15 +55,22 @@ public final class ReductionSignature<V> {
*/
private final List<ChildDescriptor<V>> childDescriptors;
/**
* Whether the represented node accepts any remaining lookup input.
*/
private final boolean acceptsRemainingInput;
/**
* Creates a signature.
*
* @param localDescriptor local descriptor
* @param childDescriptors child descriptors
*/
private ReductionSignature(final Object localDescriptor, final List<ChildDescriptor<V>> childDescriptors) {
private ReductionSignature(final Object localDescriptor, final List<ChildDescriptor<V>> childDescriptors,
final boolean acceptsRemainingInput) {
this.localDescriptor = localDescriptor;
this.childDescriptors = childDescriptors;
this.acceptsRemainingInput = acceptsRemainingInput;
}
/**
@@ -72,11 +79,14 @@ public final class ReductionSignature<V> {
* @param localSummary local value summary
* @param children reduced children
* @param settings reduction settings
* @param acceptsRemainingInput whether this node accepts any remaining lookup
* input
* @param <V> value type
* @return subtree signature
*/
public static <V> ReductionSignature<V> create(final LocalValueSummary<V> localSummary,
final Map<Character, ReducedNode<V>> children, final ReductionSettings settings) {
final Map<Character, ReducedNode<V>> children, final ReductionSettings settings,
final boolean acceptsRemainingInput) {
final Object localDescriptor = switch (settings.reductionMode()) {
case MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS ->
RankedLocalDescriptor.of(localSummary.orderedValues());
@@ -100,12 +110,28 @@ public final class ReductionSignature<V> {
childDescriptors.add(new ChildDescriptor<>(entry.getKey(), entry.getValue().signature()));
}
return new ReductionSignature<>(localDescriptor, Collections.unmodifiableList(childDescriptors));
return new ReductionSignature<>(localDescriptor, Collections.unmodifiableList(childDescriptors),
acceptsRemainingInput);
}
/**
* Creates a non-accepting subtree signature according to the selected reduction
* mode.
*
* @param localSummary local value summary
* @param children reduced children
* @param settings reduction settings
* @param <V> value type
* @return subtree signature
*/
public static <V> ReductionSignature<V> create(final LocalValueSummary<V> localSummary,
final Map<Character, ReducedNode<V>> children, final ReductionSettings settings) {
return create(localSummary, children, settings, false);
}
@Override
public int hashCode() {
return Objects.hash(this.localDescriptor, this.childDescriptors);
return Objects.hash(this.localDescriptor, this.childDescriptors, this.acceptsRemainingInput);
}
@Override
@@ -118,6 +144,7 @@ public final class ReductionSignature<V> {
}
final ReductionSignature<?> that = (ReductionSignature<?>) other;
return Objects.equals(this.localDescriptor, that.localDescriptor)
&& Objects.equals(this.childDescriptors, that.childDescriptors);
&& Objects.equals(this.childDescriptors, that.childDescriptors)
&& this.acceptsRemainingInput == that.acceptsRemainingInput;
}
}

1
src/test/java/org/egothor/stemmer/CompileIntegrationTest.java
View File

@@ -100,6 +100,7 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("slow")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@DisplayName("Compile integration")
@SuppressWarnings("deprecation")
final class CompileIntegrationTest {
/**

341
src/test/java/org/egothor/stemmer/CompiledPatchCommandTest.java Normal file
View File

@@ -0,0 +1,341 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.stream.Stream;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
/**
* Unit tests for {@link CompiledPatchCommand}.
*/
@DisplayName("CompiledPatchCommand")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
@SuppressWarnings("deprecation")
final class CompiledPatchCommandTest {
/**
* Provides representative source-target pairs for compiled command validation.
*
* @return test arguments
*/
private static Stream<Arguments> provideRoundTripPairs() {
return Stream.of(
Arguments.of(WordTraversalDirection.BACKWARD, "", ""),
Arguments.of(WordTraversalDirection.BACKWARD, "a", "a"),
Arguments.of(WordTraversalDirection.BACKWARD, "a", "b"),
Arguments.of(WordTraversalDirection.BACKWARD, "abc", "ab"),
Arguments.of(WordTraversalDirection.BACKWARD, "ab", "abc"),
Arguments.of(WordTraversalDirection.BACKWARD, "teacher", "teach"),
Arguments.of(WordTraversalDirection.BACKWARD, "running", "run"),
Arguments.of(WordTraversalDirection.BACKWARD, "cities", "city"),
Arguments.of(WordTraversalDirection.BACKWARD, "mississippi", "missouri"),
Arguments.of(WordTraversalDirection.FORWARD, "", ""),
Arguments.of(WordTraversalDirection.FORWARD, "a", "a"),
Arguments.of(WordTraversalDirection.FORWARD, "a", "b"),
Arguments.of(WordTraversalDirection.FORWARD, "abc", "bc"),
Arguments.of(WordTraversalDirection.FORWARD, "bc", "abc"),
Arguments.of(WordTraversalDirection.FORWARD, "transformation", "transform"),
Arguments.of(WordTraversalDirection.FORWARD, "cities", "city"));
}
/**
* Provides malformed compatibility patch commands.
*
* @return test arguments
*/
private static Stream<Arguments> providePreservePatchCommands() {
return Stream.of(
Arguments.of((Object) null),
Arguments.of(""),
Arguments.of("D`"),
Arguments.of("-`"),
Arguments.of("DaX"));
}
/**
* Provides compound patch commands that stress direct compiled execution.
*
* @return test arguments
*/
private static Stream<Arguments> provideCompoundPatchCommands() {
return Stream.of(
Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "IaIbIc"),
Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-bDcIxRy"),
Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "DbIxIy-cRz"),
Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-z"),
Arguments.of(WordTraversalDirection.FORWARD, "abcdef", "-zIx"),
Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "IxIyIz"),
Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-bDcIxRy"),
Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "DbIxIy-cRz"),
Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-z"),
Arguments.of(WordTraversalDirection.BACKWARD, "abcdef", "-zIx"),
Arguments.of(WordTraversalDirection.BACKWARD, "a", "DaDa"));
}
/**
* Provides patch commands whose length delta would produce an empty stem.
*
* @return test arguments
*/
private static Stream<Arguments> provideEmptyStemPatchCommands() {
return Stream.of(
Arguments.of(WordTraversalDirection.BACKWARD, "a", "Da"),
Arguments.of(WordTraversalDirection.FORWARD, "a", "Da"),
Arguments.of(WordTraversalDirection.FORWARD, "a", "DaDa"));
}
/**
* Verifies that representative serialized commands compile to concrete command
* classes instead of one universal runtime-dispatched command shape.
*/
@Test
@DisplayName("compiles representative commands to concrete command classes")
void shouldCompileRepresentativeCommandsToConcreteClasses() {
assertAll(
() -> assertEquals("DeleteSuffixCommand",
CompiledPatchCommand.compile("Da", WordTraversalDirection.BACKWARD)
.getClass().getSimpleName()),
() -> assertEquals("DeletePrefixCommand",
CompiledPatchCommand.compile("Da", WordTraversalDirection.FORWARD)
.getClass().getSimpleName()),
() -> assertEquals("AppendCharacterCommand",
CompiledPatchCommand.compile("Ix", WordTraversalDirection.BACKWARD)
.getClass().getSimpleName()),
() -> assertEquals("PrependCharacterCommand",
CompiledPatchCommand.compile("Ix", WordTraversalDirection.FORWARD)
.getClass().getSimpleName()),
() -> assertEquals("ReplaceLastCharacterCommand",
CompiledPatchCommand.compile("Rx", WordTraversalDirection.BACKWARD)
.getClass().getSimpleName()),
() -> assertEquals("ReplaceFirstCharacterCommand",
CompiledPatchCommand.compile("Rx", WordTraversalDirection.FORWARD)
.getClass().getSimpleName()),
() -> assertEquals("BackwardCompoundCommand",
CompiledPatchCommand.compile("-aDa", WordTraversalDirection.BACKWARD)
.getClass().getSimpleName()),
() -> assertEquals("ForwardCompoundCommand",
CompiledPatchCommand.compile("-aDa", WordTraversalDirection.FORWARD)
.getClass().getSimpleName()));
}
/**
* Verifies the no-op fast-path marker used by high-throughput callers.
*/
@Test
@DisplayName("marks only all-source preserve commands as preserving")
void shouldMarkOnlyAllSourcePreserveCommandsAsPreserving() {
assertAll(
() -> assertEquals(true,
CompiledPatchCommand.compile(null, WordTraversalDirection.BACKWARD).preservesAllSources()),
() -> assertEquals(true,
CompiledPatchCommand.compile("", WordTraversalDirection.BACKWARD).preservesAllSources()),
() -> assertEquals(true,
CompiledPatchCommand.compile("Na", WordTraversalDirection.BACKWARD).preservesAllSources()),
() -> assertEquals(true,
CompiledPatchCommand.compile("-a", WordTraversalDirection.BACKWARD).preservesAllSources()),
() -> assertEquals(false,
CompiledPatchCommand.compile("Da", WordTraversalDirection.BACKWARD).preservesAllSources()),
() -> assertEquals(false,
CompiledPatchCommand.compile("Ix", WordTraversalDirection.FORWARD).preservesAllSources()),
() -> assertEquals(false,
CompiledPatchCommand.compile("-aDa", WordTraversalDirection.BACKWARD)
.preservesAllSources()));
}
/**
* Verifies that compiled commands match the string interpreter.
*
* @param traversalDirection traversal direction
* @param source source word
* @param target target word
*/
@ParameterizedTest
@MethodSource("provideRoundTripPairs")
@DisplayName("matches interpreted patch application")
void shouldMatchInterpretedPatchApplication(final WordTraversalDirection traversalDirection, final String source,
final String target) {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
.traversalDirection(traversalDirection)
.build();
final String patch = encoder.encode(source, target);
final CompiledPatchCommand compiled = encoder.compile(patch);
final String expected = PatchCommandEncoder.apply(source, patch, traversalDirection);
final char[] sequenceOutput = new char[Math.max(source.length(), expected.length()) + 8];
final int sequenceLength = compiled.applyTo(source, sequenceOutput, 2, sequenceOutput.length - 2);
final char[] sourceArray = source.toCharArray();
final char[] arrayOutput = new char[Math.max(source.length(), expected.length()) + 8];
final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 1,
arrayOutput.length - 1);
assertAll(
() -> assertEquals(expected, compiled.apply(source)),
() -> assertEquals(expected.length(), sequenceLength),
() -> assertEquals(expected, new String(sequenceOutput, 2, sequenceLength)),
() -> assertEquals(expected.length(), arrayLength),
() -> assertEquals(expected, new String(arrayOutput, 1, arrayLength)));
}
/**
* Verifies compound direct execution against the compatibility interpreter.
*
* @param traversalDirection traversal direction
* @param source source word
* @param patch serialized compound patch command
*/
@ParameterizedTest
@MethodSource("provideCompoundPatchCommands")
@DisplayName("matches interpreted compound patch application")
void shouldMatchInterpretedCompoundPatchApplication(final WordTraversalDirection traversalDirection,
final String source, final String patch) {
final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patch, traversalDirection);
final String expected = PatchCommandEncoder.apply(source, patch, traversalDirection);
final char[] sequenceOutput = new char[Math.max(source.length(), expected.length()) + 8];
final char[] sourceArray = source.toCharArray();
final char[] arrayOutput = new char[Math.max(source.length(), expected.length()) + 8];
final int sequenceLength = compiled.applyTo(source, sequenceOutput, 2, sequenceOutput.length - 2);
final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 1,
arrayOutput.length - 1);
assertAll(
() -> assertEquals(expected, compiled.apply(source)),
() -> assertEquals(expected.length(), sequenceLength),
() -> assertEquals(expected, new String(sequenceOutput, 2, sequenceLength)),
() -> assertEquals(expected.length(), arrayLength),
() -> assertEquals(expected, new String(arrayOutput, 1, arrayLength)));
}
/**
* Verifies that the compiled hot path never produces an empty stem.
*
* @param traversalDirection traversal direction
* @param source source word
* @param patch serialized patch command
*/
@ParameterizedTest
@MethodSource("provideEmptyStemPatchCommands")
@DisplayName("preserves source when a patch would produce an empty stem")
void shouldPreserveSourceWhenPatchWouldProduceEmptyStem(final WordTraversalDirection traversalDirection,
final String source, final String patch) {
final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patch, traversalDirection);
final char[] sequenceOutput = new char[source.length() + 4];
final char[] sourceArray = source.toCharArray();
final char[] arrayOutput = new char[source.length() + 4];
final int sequenceLength = compiled.applyTo(source, sequenceOutput, 1, sequenceOutput.length - 1);
final int arrayLength = compiled.applyTo(sourceArray, 0, sourceArray.length, arrayOutput, 2,
arrayOutput.length - 2);
assertAll(
() -> assertSame(source, compiled.apply(source)),
() -> assertEquals(source.length(), sequenceLength),
() -> assertEquals(source, new String(sequenceOutput, 1, sequenceLength)),
() -> assertEquals(source.length(), arrayLength),
() -> assertEquals(source, new String(arrayOutput, 2, arrayLength)));
}
/**
* Verifies preserve-only patch commands.
*
* @param patchCommand serialized patch command
*/
@ParameterizedTest
@MethodSource("providePreservePatchCommands")
@DisplayName("preserves source for interpreted preserve-only commands")
void shouldPreserveSourceForPreserveOnlyCommands(final String patchCommand) {
final String source = "teacher";
final CompiledPatchCommand compiled = CompiledPatchCommand.compile(patchCommand, WordTraversalDirection.BACKWARD);
assertSame(source, compiled.apply(source));
}
/**
* Verifies insufficient output capacity reporting.
*/
@ParameterizedTest
@MethodSource("provideRoundTripPairs")
@DisplayName("reports insufficient output capacity")
void shouldReportInsufficientOutputCapacity(final WordTraversalDirection traversalDirection, final String source,
final String target) {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
.traversalDirection(traversalDirection)
.build();
final String patch = encoder.encode(source, target);
final CompiledPatchCommand compiled = encoder.compile(patch);
final String expected = compiled.apply(source);
final char[] output = new char[Math.max(0, expected.length() - 1)];
if (expected.isEmpty()) {
assertEquals(0, compiled.applyTo(source, output, 0, output.length));
} else {
assertEquals(CompiledPatchCommand.APPLY_INSUFFICIENT_CAPACITY,
compiled.applyTo(source, output, 0, output.length));
}
}
/**
* Verifies compile-time rejection of unsupported serialized commands.
*/
@ParameterizedTest
@MethodSource("provideInvalidPatchCommands")
@DisplayName("rejects unsupported patch commands")
void shouldRejectUnsupportedPatchCommands(final String patchCommand) {
assertThrows(IllegalArgumentException.class,
() -> CompiledPatchCommand.compile(patchCommand, WordTraversalDirection.BACKWARD));
}
/**
* Provides invalid patch commands.
*
* @return test arguments
*/
private static Stream<Arguments> provideInvalidPatchCommands() {
return Stream.of(
Arguments.of("Xa"),
Arguments.of("N`"),
Arguments.of("DaN`"));
}
}

1
src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java
View File

@@ -76,6 +76,7 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("serialization")
@Tag("trie")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@SuppressWarnings("deprecation")
final class CompiledTrieArtifactRegressionTest {
/**

33
src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java
View File

@@ -269,6 +269,25 @@ class FrequencyTrieBuildersTest {
assertTrieStateEquals(original, reconstructed, "xy");
}
/**
* Verifies that compiled trie values can be mapped to another value type while
* preserving lookup semantics and local counts.
*/
@Test
@DisplayName("should map values while preserving keys and counts")
void shouldMapValuesWhilePreservingKeysAndCounts() {
final FrequencyTrie<String> original = createRepresentativeTrie();
final FrequencyTrie<String> mapped = FrequencyTrieBuilders.mapValues(original, ARRAY_FACTORY,
RANKED_SETTINGS, value -> "mapped-" + value);
assertAll(
() -> assertEquals("mapped-root-main", mapped.get("")),
() -> assertArrayEquals(new String[] { "mapped-A1", "mapped-A2" }, mapped.getAll("a")),
() -> assertIterableEquals(List.of(new ValueCount<String>("mapped-AB1", 5),
new ValueCount<String>("mapped-AB2", 2)), mapped.getEntries("ab")));
}
/**
* Verifies the documented null-argument contract for both public reconstruction
* entry points.
@@ -292,7 +311,19 @@ class FrequencyTrieBuildersTest {
() -> FrequencyTrieBuilders.copyOf(trie, null,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.copyOf(trie, ARRAY_FACTORY, (ReductionMode) null)));
() -> FrequencyTrieBuilders.copyOf(trie, ARRAY_FACTORY, (ReductionMode) null)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.mapValues(null, Integer[]::new, RANKED_SETTINGS, String::length)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.mapValues(trie, null, RANKED_SETTINGS, String::length)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, (ReductionSettings) null,
String::length)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, RANKED_SETTINGS, null)),
() -> assertThrows(NullPointerException.class,
() -> FrequencyTrieBuilders.mapValues(trie, Integer[]::new, (ReductionMode) null,
String::length)));
}
/**

118
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -94,6 +94,17 @@ class FrequencyTrieTest {
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
}
/**
* Creates reduction settings with the internal uniform-subtree contraction
* enabled.
*
* @return contraction-enabled settings
*/
private static ReductionSettings uniformSubtreeContractionSettings() {
return ReductionSettings.withUniformSubtreeContraction(ReductionSettings
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
}
/**
* Verifies that the builder rejects {@code null} constructor arguments.
*/
@@ -508,6 +519,10 @@ class FrequencyTrieTest {
() -> trie.getAllNormalized((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized("house", null, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getNormalized(null)),
() -> assertThrows(NullPointerException.class,
() -> trie.getNormalizedString(null)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAll((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
@@ -537,6 +552,12 @@ class FrequencyTrieTest {
}, 10)),
() -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true),
"Normalized lookup must bypass metadata lowercasing."),
() -> assertNull(trie.getNormalized("HOUSE"),
"Normalized preferred lookup must bypass metadata lowercasing."),
() -> assertNull(trie.getNormalizedString("HOUSE"),
"String-specialized normalized lookup must bypass metadata lowercasing."),
() -> assertEquals("noun", trie.getNormalized("house")),
() -> assertEquals("noun", trie.getNormalizedString("house")),
() -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> {
assertEquals("noun", value);
return true;
@@ -1056,6 +1077,103 @@ class FrequencyTrieTest {
() -> assertEquals(original.get("z"), disabledDense.get("z")));
}
/**
* Verifies that uniform subtree contraction is not part of the default generic
* trie semantics.
*/
@Test
@Tag("reduction")
@DisplayName("Default reduction keeps exact lookup semantics for uniform subtrees")
void shouldKeepExactLookupWhenUniformSubtreeContractionIsDisabled() {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
WordTraversalDirection.FORWARD);
builder.put("aa", "x");
builder.put("ab", "x");
final FrequencyTrie<String> trie = builder.build();
assertAll("exact lookup",
() -> assertEquals("x", trie.get("aa")),
() -> assertEquals("x", trie.get("ab")),
() -> assertNull(trie.get("az")),
() -> assertFalse(trie.root().findChild('a').acceptsRemainingInput()));
}
/**
* Verifies that the internal uniform-subtree contraction replaces a uniform
* non-leaf subtree with an accepting leaf.
*/
@Test
@Tag("reduction")
@DisplayName("Uniform subtree contraction replaces uniform internal subtree with accepting leaf")
void shouldContractUniformInternalSubtreeIntoAcceptingLeaf() {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD);
builder.put("aa", "x");
builder.put("ab", "x");
builder.put("ba", "y");
final FrequencyTrie<String> trie = builder.build();
assertAll("contracted lookup",
() -> assertEquals(3, trie.size()),
() -> assertTrue(trie.root().findChild('a').acceptsRemainingInput()),
() -> assertEquals("x", trie.get("a")),
() -> assertEquals("x", trie.get("aa")),
() -> assertEquals("x", trie.get("ab")),
() -> assertEquals("x", trie.get("az")),
() -> assertEquals("y", trie.get("bz")),
() -> assertNull(trie.get("c")));
}
/**
* Verifies that binary persistence preserves accepting leaf semantics.
*/
@Test
@Tag("persistence")
@DisplayName("Binary round trip preserves uniform subtree accepting leaf")
void shouldPreserveUniformSubtreeContractionAcrossBinaryRoundTrip() throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD);
builder.put("aa", "x");
builder.put("ab", "x");
builder.put("ba", "y");
final FrequencyTrie<String> original = builder.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
original.writeTo(outputStream, STRING_CODEC);
final FrequencyTrie<String> restored = FrequencyTrie
.readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
assertAll("restored contraction",
() -> assertTrue(restored.root().findChild('a').acceptsRemainingInput()),
() -> assertEquals("x", restored.get("az")),
() -> assertTrue(restored.metadata().reductionSettings().contractUniformSubtrees()));
}
/**
* Verifies that value mapping keeps accepting leaf semantics.
*/
@Test
@Tag("reduction")
@DisplayName("Value mapping preserves uniform subtree accepting leaf")
void shouldPreserveUniformSubtreeContractionWhenMappingValues() {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
uniformSubtreeContractionSettings(), WordTraversalDirection.FORWARD);
builder.put("aa", "x");
builder.put("ab", "x");
builder.put("ba", "y");
final FrequencyTrie<String> source = builder.build();
final FrequencyTrie<Integer> mapped = FrequencyTrieBuilders.mapValues(source, Integer[]::new,
source.metadata().reductionSettings(), String::length);
assertAll("mapped contraction",
() -> assertTrue(mapped.root().findChild('a').acceptsRemainingInput()),
() -> assertEquals(1, mapped.get("az")));
}
/**
* Verifies that cyclic serialized node references are rejected as invalid
* serialization.

1
src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java
View File

@@ -69,6 +69,7 @@ import org.junit.jupiter.api.io.TempDir;
@Tag("trie")
@Tag("stemmer")
@Tag("determinism")
@SuppressWarnings("deprecation")
class FuzzStemmerAndTrieCompilationTest {
/**

1
src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java
View File

@@ -50,6 +50,7 @@ import org.junit.jupiter.api.Tag;
@Tag("property")
@Tag("patch")
@Tag("stemmer")
@SuppressWarnings("deprecation")
class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
/**

1
src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java
View File

@@ -72,6 +72,7 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("encoding")
@Tag("apply")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@SuppressWarnings("deprecation")
class PatchCommandEncoderTest {
/**

1
src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java
View File

@@ -56,6 +56,7 @@ import java.util.logging.Logger;
* <li>{@code --reduction-mode <enum-name>}</li>
* </ul>
*/
@SuppressWarnings("deprecation")
public final class RegressionArtifactGenerator {
/**

1
src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java
View File

@@ -49,6 +49,7 @@ import java.util.Objects;
* calculation, and failure-message formatting so that regression tests stay
* focused on contract verification.
*/
@SuppressWarnings("deprecation")
final class RegressionArtifactSupport {
/**

157
src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java
View File

@@ -92,6 +92,7 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("trie")
@Tag("persistence")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@SuppressWarnings("deprecation")
final class StemmerPatchTrieLoaderTest {
/**
@@ -251,6 +252,72 @@ final class StemmerPatchTrieLoaderTest {
StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("27-load-binary-metadata-stream-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
"inputStream"),
Arguments.of("28-load-compiled-language-settings-null-language",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(
(StemmerPatchTrieLoader.Language) null, true, settings),
"language"),
Arguments.of("29-load-compiled-language-settings-null-settings",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.US_UK, true, (ReductionSettings) null),
"reductionSettings"),
Arguments.of("30-load-compiled-language-mode-null-language",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(
(StemmerPatchTrieLoader.Language) null, true, DEFAULT_REDUCTION_MODE),
"language"),
Arguments.of("31-load-compiled-language-mode-null-mode",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.US_UK, true, (ReductionMode) null),
"reductionMode"),
Arguments.of("32-load-compiled-language-metadata-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.US_UK, true, (TrieMetadata) null),
"metadata"),
Arguments.of("33-load-compiled-path-settings-null-path",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled((Path) null, true, settings),
"path"),
Arguments.of("34-load-compiled-path-settings-null-settings",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true,
(ReductionSettings) null),
"reductionSettings"),
Arguments.of("35-load-compiled-path-mode-null-mode",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true,
(ReductionMode) null),
"reductionMode"),
Arguments.of("36-load-compiled-path-metadata-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath(), true,
(TrieMetadata) null),
"metadata"),
Arguments.of("37-load-compiled-string-settings-null-file",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled((String) null, true, settings),
StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("38-load-compiled-string-settings-null-settings",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true,
(ReductionSettings) null),
"reductionSettings"),
Arguments.of("39-load-compiled-string-mode-null-mode",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true,
(ReductionMode) null),
"reductionMode"),
Arguments.of("40-load-compiled-string-metadata-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadCompiled(tempPath().toString(), true,
(TrieMetadata) null),
"metadata"),
Arguments.of("41-load-binary-compiled-path-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((Path) null), "path"),
Arguments.of("42-load-binary-compiled-path-override-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((Path) null,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"path"),
Arguments.of("43-load-binary-compiled-string-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((String) null),
StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("44-load-binary-compiled-string-override-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((String) null,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("45-load-binary-compiled-stream-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryCompiled((InputStream) null),
"inputStream"));
}
@@ -431,6 +498,38 @@ final class StemmerPatchTrieLoaderTest {
"run");
}
/**
* Verifies that textual compiled loading overloads produce patch-command
* tries with the same stemming semantics as the canonical textual patch trie.
*
* @throws IOException if the test file cannot be written or read
*/
@Test
@DisplayName("Textual compiled load overloads must preserve stemming semantics")
void shouldLoadCompiledTrieFromTextualOverloads() throws IOException {
final Path dictionaryFile = writeDictionary("""
run running runs runner
play playing played plays
city cities
""");
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
final FrequencyTrie<String> expected = StemmerPatchTrieLoader.load(dictionaryFile, true, settings);
final FrequencyTrie<CompiledPatchCommand> fromPathWithSettings = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile, true, settings);
final FrequencyTrie<CompiledPatchCommand> fromPathWithMode = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile, true, DEFAULT_REDUCTION_MODE);
final FrequencyTrie<CompiledPatchCommand> fromStringWithSettings = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile.toString(), true, settings);
final FrequencyTrie<CompiledPatchCommand> fromStringWithMode = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile.toString(), true, DEFAULT_REDUCTION_MODE);
assertCompiledTrieSemanticsEqual(expected, fromPathWithSettings, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromPathWithMode, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromStringWithSettings, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromStringWithMode, "running", "played", "cities", "run");
}
/**
* Verifies that metadata-driven loading keeps all configuration dimensions in
* one explicit object and applies them during compilation.
@@ -577,14 +676,28 @@ final class StemmerPatchTrieLoaderTest {
StemmerPatchTrieLoader.saveBinary(original, binaryFile);
final FrequencyTrie<String> fromPath = StemmerPatchTrieLoader.loadBinary(binaryFile);
final FrequencyTrie<String> fromString = StemmerPatchTrieLoader.loadBinary(binaryFile.toString());
final FrequencyTrie<CompiledPatchCommand> compiledFromPath = StemmerPatchTrieLoader.loadBinaryCompiled(
binaryFile);
final FrequencyTrie<CompiledPatchCommand> compiledFromString = StemmerPatchTrieLoader.loadBinaryCompiled(
binaryFile.toString());
final byte[] binaryBytes = Files.readAllBytes(binaryFile);
try (InputStream inputStream = new ByteArrayInputStream(binaryBytes)) {
final FrequencyTrie<String> fromStream = StemmerPatchTrieLoader.loadBinary(inputStream);
final FrequencyTrie<CompiledPatchCommand> compiledFromStream;
try (InputStream compiledInputStream = new ByteArrayInputStream(binaryBytes)) {
compiledFromStream = StemmerPatchTrieLoader.loadBinaryCompiled(compiledInputStream);
}
assertTriePatchSemanticsEqual(original, fromPath, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromString, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromStream, "run", "running", "runner", "cities", "studying");
assertCompiledTrieSemanticsEqual(original, compiledFromPath, "run", "running", "runner", "cities",
"studying");
assertCompiledTrieSemanticsEqual(original, compiledFromString, "run", "running", "runner", "cities",
"studying");
assertCompiledTrieSemanticsEqual(original, compiledFromStream, "run", "running", "runner", "cities",
"studying");
}
final TrieMetadata metadataFromPath = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile);
@@ -856,6 +969,30 @@ final class StemmerPatchTrieLoaderTest {
return stems;
}
/**
* Reconstructs all stem candidates for the supplied word from compiled patch
* commands returned by {@link FrequencyTrie#getAll(String)}.
*
* @param trie compiled patch-command trie
* @param word surface word
* @return reconstructed stem candidates
*/
private static Set<String> reconstructAllCompiledStemCandidates(final FrequencyTrie<CompiledPatchCommand> trie,
final String word) {
final CompiledPatchCommand[] patchCommands = trie.getAll(word);
final Set<String> stems = new LinkedHashSet<String>();
if (patchCommands == null) {
return stems;
}
for (CompiledPatchCommand patchCommand : patchCommands) {
stems.add(patchCommand.apply(word));
}
return stems;
}
/**
* Verifies semantic equality of two tries for the supplied words by comparing
* both their raw patch arrays and reconstructed stem sets.
@@ -876,6 +1013,26 @@ final class StemmerPatchTrieLoaderTest {
}
}
/**
* Verifies semantic equality of a textual patch trie and a compiled patch trie
* for the supplied words.
*
* @param expected reference trie with textual patch commands
* @param actual compared trie with compiled patch commands
* @param words words to verify
*/
private static void assertCompiledTrieSemanticsEqual(final FrequencyTrie<String> expected,
final FrequencyTrie<CompiledPatchCommand> actual, final String... words) {
assertAll(() -> assertEquals(expected.metadata(), actual.metadata(), "Trie metadata must be preserved."),
() -> assertEquals(expected.traversalDirection(), actual.traversalDirection(),
"Trie traversal direction must be preserved."));
for (String word : words) {
assertEquals(reconstructAllStemCandidates(expected, word), reconstructAllCompiledStemCandidates(actual, word),
"Compiled patch stems must match textual patch stems for word '" + word + "'.");
}
}
/**
* Opens one bundled dictionary resource.
*

1
src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java
View File

@@ -58,6 +58,7 @@ import org.junit.jupiter.api.Tag;
@Label("Stemmer patch trie properties")
@Tag("property")
@Tag("stemmer")
@SuppressWarnings("deprecation")
class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
/**

187
src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java Normal file
View File

@@ -0,0 +1,187 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Unit tests for benchmark token sequence reuse and offset behavior.
*/
@Tag("benchmark")
@Tag("unit")
@DisplayName("BenchmarkTokenSequence")
final class BenchmarkTokenSequenceTest {
/**
* Shared token corpus used in the benchmark state.
*/
private static final String[] TOKENS = { "running", "caresses", "running", "happiness", "", "s" };
/**
* Fully qualified benchmark helper class name.
*/
private static final String SEQUENCE_CLASS = "org.egothor.stemmer.benchmark.BenchmarkTokenSequence";
/**
* Verifies first run and rewind behavior keeps token order stable.
*/
@Test
@DisplayName("should emit tokens in stable order and correct offsets")
void shouldEmitTokensInStableOrder() throws Exception {
final Object sequence = createSequence(TOKENS);
final String[] expected = TOKENS;
final Method advance = method("advance");
final Method hasNext = method("hasNext");
final Method currentToken = method("currentToken");
final Method currentStartOffset = method("currentStartOffset");
final Method currentEndOffset = method("currentEndOffset");
for (int index = 0; index < expected.length; index++) {
assertTrue((Boolean) advance.invoke(sequence));
assertEquals(expected[index], new String((char[]) currentToken.invoke(sequence)));
if (index == 0) {
assertEquals(0, ((Number) currentStartOffset.invoke(sequence)).intValue());
assertEquals("running".length(), ((Number) currentEndOffset.invoke(sequence)).intValue());
}
}
assertFalse((Boolean) hasNext.invoke(sequence));
assertFalse((Boolean) advance.invoke(sequence));
}
/**
* Verifies reset brings the sequence back to the first token.
*/
@Test
@DisplayName("should support reset and replay without allocations")
void shouldSupportResetAndReplay() throws Exception {
final Object sequence = createSequence(TOKENS);
final Method advance = method("advance");
final Method reset = method("reset");
int firstPass = 0;
while ((Boolean) advance.invoke(sequence)) {
firstPass++;
}
reset.invoke(sequence);
int secondPass = 0;
while ((Boolean) advance.invoke(sequence)) {
secondPass++;
}
assertEquals(firstPass, secondPass);
}
/**
* Verifies sequence tokens can be replaced for multi-benchmark reuse.
*/
@Test
@DisplayName("should reset offsets after token set replacement")
void shouldResetOffsetsAfterTokenReplacement() throws Exception {
final Object sequence = createSequence(new String[] { "a", "bc", "def" });
final Method setTokens = method("setTokens", String[].class);
final Method advance = method("advance");
final Method currentToken = method("currentToken");
advance.invoke(sequence);
advance.invoke(sequence);
final String[] replacement = { "xy", "z" };
setTokens.invoke(sequence, (Object) replacement);
advance.invoke(sequence);
assertEquals("xy", new String((char[]) currentToken.invoke(sequence)));
advance.invoke(sequence);
assertEquals("z", new String((char[]) currentToken.invoke(sequence)));
}
/**
* Verifies no mutation of original token instances in source corpus.
*/
@Test
@DisplayName("should not mutate source token references or values")
void shouldNotMutateSourceTokenValues() throws Exception {
final String[] source = { "first", "second", "third" };
final Object sequence = createSequence(source);
final Method advance = method("advance");
final Method currentToken = method("currentToken");
while ((Boolean) advance.invoke(sequence)) {
assertFalse(new String((char[]) currentToken.invoke(sequence)).isEmpty());
}
assertEquals("first", source[0]);
assertEquals("second", source[1]);
assertEquals("third", source[2]);
}
/**
* Instantiates the benchmark token sequence class.
*
* @param tokens source tokens
* @return sequence instance
* @throws Exception on reflection failure
*/
private Object createSequence(final String[] tokens) throws Exception {
try {
final Class<?> type = Class.forName(SEQUENCE_CLASS);
final Constructor<?> constructor = type.getDeclaredConstructor(String[].class);
return constructor.newInstance((Object) tokens);
} catch (ClassNotFoundException exception) {
Assumptions.assumeTrue(false, "Benchmark token sequence class is available only when JMH sources are compiled.");
throw exception;
}
}
/**
* Resolves a method for the token sequence class.
*
* @param name method name
* @param arguments argument types
* @return method
* @throws Exception when method is missing
*/
private Method method(final String name, final Class<?>... arguments) throws Exception {
final Method method = Class.forName(SEQUENCE_CLASS).getDeclaredMethod(name, arguments);
method.setAccessible(true);
return method;
}
}

230
src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java Normal file
View File

@@ -0,0 +1,230 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Reflection-based tests for the Lucene-dependent benchmark token stream.
*/
@Tag("benchmark")
@Tag("unit")
@DisplayName("EnglishStemmerComparisonTokenStream")
final class EnglishStemmerComparisonTokenStreamTest {
/**
* Fully qualified token stream class name.
*/
private static final String TOKEN_STREAM_CLASS = "org.egothor.stemmer.benchmark.EnglishStemmerComparisonTokenStream";
/**
* Verifies stream reuse and reset behavior through repeated iteration.
*
* @throws Exception when reflection calls fail
*/
@Test
@DisplayName("should reuse token stream without losing order or count")
void shouldReuseTokenStreamWithoutLosingOrderOrCount() throws Exception {
final Object stream = createStream(new String[] { "running", "caresses", "happiness" });
final Class<?> type = streamType();
final Method increment = method(type, "incrementToken");
final Method reset = method(type, "reset");
final Method isDrained = method(type, "isDrained");
final Method setTokens = method(type, "setTokens", String[].class);
int count = consume(increment, stream);
assertEquals(3, count);
assertEquals(Boolean.TRUE, isDrained.invoke(stream));
reset.invoke(stream);
assertEquals(3, consume(increment, stream));
setTokens.invoke(stream, (Object) new String[] { "single" });
reset.invoke(stream);
assertEquals(1, consume(increment, stream));
}
/**
* Verifies empty stream handling, end-of-stream, and reset behavior.
*
* @throws Exception when reflection calls fail
*/
@Test
@DisplayName("should handle empty corpus with immediate drain")
void shouldHandleEmptyCorpusWithImmediateDrain() throws Exception {
final Object stream = createStream(new String[0]);
final Class<?> type = streamType();
final Method increment = method(type, "incrementToken");
final Method isDrained = method(type, "isDrained");
final Method reset = method(type, "reset");
assertFalse((Boolean) increment.invoke(stream));
assertEquals(Boolean.TRUE, isDrained.invoke(stream));
reset.invoke(stream);
assertFalse((Boolean) increment.invoke(stream));
}
/**
* Verifies the last emitted token is stable across repeated passes.
*
* @throws Exception when reflection calls fail
*/
@Test
@DisplayName("should expose stable terminal token text")
void shouldExposeStableTerminalTokenText() throws Exception {
final Object stream = createStream(new String[] { "caresses", "running", "connected" });
final Class<?> type = streamType();
final Class<?> charTermClass = Class.forName("org.apache.lucene.analysis.tokenattributes.CharTermAttribute");
final Method increment = method(type, "incrementToken");
final Method getAttribute = method(type, "getAttribute", false, Class.class);
final Method end = method(type, "end");
final Method close = method(type, "close");
final Object termAttribute = getAttribute.invoke(stream, charTermClass);
String lastToken = null;
while ((Boolean) increment.invoke(stream)) {
lastToken = termAttribute.toString();
}
end.invoke(stream);
close.invoke(stream);
assertEquals("connected", lastToken);
}
/**
* Resolves and instantiate the benchmark token stream class when available.
*
* @param tokens input tokens
* @return created stream
* @throws Exception when class or constructor fails
*/
private Object createStream(final String[] tokens) throws Exception {
final Class<?> type = streamType();
final Constructor<?> constructor = type.getDeclaredConstructor(String[].class);
constructor.setAccessible(true);
return constructor.newInstance((Object) tokens);
}
/**
* Resolves the benchmark token stream class.
*
* @return stream class
* @throws Exception when class loading fails
*/
private Class<?> streamType() throws Exception {
try {
return Class.forName(TOKEN_STREAM_CLASS);
} catch (ClassNotFoundException exception) {
Assumptions.assumeTrue(false, "Token stream class is available only when JMH sources are compiled.");
throw exception;
}
}
/**
* Resolves a method for invocation.
*
* @param type target class
* @param name method name
* @param arguments argument types
* @return reflected method
* @throws NoSuchMethodException when method is missing
*/
private Method method(final Class<?> type, final String name, final Class<?>... arguments) throws NoSuchMethodException {
return method(type, name, true, arguments);
}
/**
* Resolves a method for invocation.
*
* @param type target class
* @param name method name
* @param declared whether to require declaration in the target class
* @param arguments argument types
* @return reflected method
* @throws NoSuchMethodException when method is missing
*/
private Method method(final Class<?> type, final String name, final boolean declared, final Class<?>... arguments)
throws NoSuchMethodException {
final Method method = declared ? type.getDeclaredMethod(name, arguments) : type.getMethod(name, arguments);
method.setAccessible(true);
return method;
}
/**
* Consumes stream and counts tokens.
*
* @param increment increment method
* @param stream stream object
* @return tokens emitted
* @throws Exception on reflection failure
*/
private int consume(final Method increment, final Object stream) throws Exception {
int count = 0;
while ((Boolean) increment.invoke(stream)) {
count++;
}
return count;
}
/**
* Confirms invocation paths fail fast for invalid signatures.
*/
@Test
@DisplayName("should enforce method contract on reflection")
void shouldEnforceMethodContractOnReflection() {
Assumptions.assumeTrue(streamTypeAvailable(), "Token stream class is available only when JMH sources are compiled.");
assertThrows(NoSuchMethodException.class, () -> streamType().getDeclaredMethod("nonExistentMethod"));
}
/**
* Checks whether the benchmark token stream class can be loaded.
*
* @return true when class is available
*/
private boolean streamTypeAvailable() {
try {
Class.forName(TOKEN_STREAM_CLASS);
return true;
} catch (ClassNotFoundException exception) {
return false;
}
}
}

228
src/test/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpusTest.java Normal file
View File

@@ -0,0 +1,228 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.lang.reflect.Method;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Tests dictionary-derived benchmark corpus construction.
*/
@Tag("benchmark")
@Tag("unit")
@DisplayName("LanguageBenchmarkCorpus")
final class LanguageBenchmarkCorpusTest {
/**
* Fully qualified corpus helper class name.
*/
private static final String CORPUS_CLASS = "org.egothor.stemmer.benchmark.LanguageBenchmarkCorpus";
/**
* Verifies large resources use the full dictionary-derived token sequence.
*
* @throws Exception if reflection or resource loading fails
*/
@Test
@DisplayName("should use full dictionary corpus when resource is larger than the timing minimum")
void shouldUseFullDictionaryCorpusWhenResourceIsLargerThanTimingMinimum() throws Exception {
final Object fullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object timingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK);
assertTrue(tokens(fullCorpus).length > minimumTimingTokenCount());
assertEquals(tokens(fullCorpus).length, tokens(timingCorpus).length);
assertArrayEquals(tokens(fullCorpus), tokens(timingCorpus));
assertArrayEquals(expectedRoots(fullCorpus), expectedRoots(timingCorpus));
}
/**
* Verifies small resources repeat deterministically to the minimum timing size.
*
* @throws Exception if reflection or resource loading fails
*/
@Test
@DisplayName("should repeat small dictionary corpus to timing minimum")
void shouldRepeatSmallDictionaryCorpusToTimingMinimum() throws Exception {
final Object fullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.FA_IR);
final Object timingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.FA_IR);
assertTrue(tokens(fullCorpus).length < minimumTimingTokenCount());
assertEquals(minimumTimingTokenCount(), tokens(timingCorpus).length);
assertEquals(tokens(fullCorpus)[0], tokens(timingCorpus)[0]);
assertEquals(expectedRoots(fullCorpus)[0], expectedRoots(timingCorpus)[0]);
assertEquals(tokens(fullCorpus)[0], tokens(timingCorpus)[tokens(fullCorpus).length]);
assertEquals(expectedRoots(fullCorpus)[0], expectedRoots(timingCorpus)[tokens(fullCorpus).length]);
}
/**
* Verifies corpus token and expected-root arrays stay aligned.
*
* @throws Exception if reflection or resource loading fails
*/
@Test
@DisplayName("should keep token and expected-root arrays aligned")
void shouldKeepTokenAndExpectedRootArraysAligned() throws Exception {
final Object corpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.PL_PL);
assertEquals(tokens(corpus).length, expectedRoots(corpus).length);
assertTrue(tokens(corpus).length > minimumTimingTokenCount());
assertTrue(expectedRoots(corpus)[0].length() > 0);
}
/**
* Verifies changed-token timing corpora exclude entries already equal to the
* expected root.
*
* @throws Exception if reflection or resource loading fails
*/
@Test
@DisplayName("should create changed-token timing corpus")
void shouldCreateChangedTokenTimingCorpus() throws Exception {
final Object corpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK);
final String[] corpusTokens = tokens(corpus);
final String[] corpusExpectedRoots = expectedRoots(corpus);
assertTrue(corpusTokens.length > minimumTimingTokenCount());
for (int index = 0; index < corpusTokens.length; index++) {
assertTrue(!corpusTokens[index].equals(corpusExpectedRoots[index]),
"Changed-token corpus must contain only token/root pairs where token differs from root.");
}
}
/**
* Verifies benchmark corpora are generated once per language and reused from
* memory.
*
* @throws Exception if reflection or resource loading fails
*/
@Test
@DisplayName("should reuse cached corpus instances")
void shouldReuseCachedCorpusInstances() throws Exception {
final Object firstTimingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object secondTimingCorpus = invokeCorpus("createCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object firstFullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object secondFullCorpus = invokeCorpus("createFullCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object firstChangedCorpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK);
final Object secondChangedCorpus = invokeCorpus("createChangedCorpus", StemmerPatchTrieLoader.Language.US_UK);
assertSame(firstTimingCorpus, secondTimingCorpus);
assertSame(firstFullCorpus, secondFullCorpus);
assertSame(firstChangedCorpus, secondChangedCorpus);
}
/**
* Invokes a static corpus factory.
*
* @param methodName factory method name
* @param language Radixor language
* @return corpus record instance
* @throws Exception if reflection fails
*/
private Object invokeCorpus(final String methodName, final StemmerPatchTrieLoader.Language language)
throws Exception {
final Class<?> type = corpusType();
final Method method = type.getDeclaredMethod(methodName, StemmerPatchTrieLoader.Language.class);
method.setAccessible(true);
return method.invoke(null, language);
}
/**
* Reads corpus tokens.
*
* @param corpus corpus record instance
* @return token array
* @throws Exception if reflection fails
*/
private String[] tokens(final Object corpus) throws Exception {
return stringArray(corpus, "tokens");
}
/**
* Reads corpus expected roots.
*
* @param corpus corpus record instance
* @return expected-root array
* @throws Exception if reflection fails
*/
private String[] expectedRoots(final Object corpus) throws Exception {
return stringArray(corpus, "expectedRoots");
}
/**
* Reads a string-array record component.
*
* @param corpus corpus record instance
* @param methodName accessor name
* @return string array
* @throws Exception if reflection fails
*/
private String[] stringArray(final Object corpus, final String methodName) throws Exception {
final Method method = corpus.getClass().getDeclaredMethod(methodName);
method.setAccessible(true);
return (String[]) method.invoke(corpus);
}
/**
* Reads the minimum timing token count constant.
*
* @return minimum timing token count
* @throws Exception if reflection fails
*/
private int minimumTimingTokenCount() throws Exception {
final java.lang.reflect.Field field = corpusType().getDeclaredField("MINIMUM_TIMING_TOKEN_COUNT");
field.setAccessible(true);
return ((Number) field.get(null)).intValue();
}
/**
* Resolves the benchmark corpus helper class.
*
* @return corpus helper type
* @throws Exception if class loading fails
*/
private Class<?> corpusType() throws Exception {
try {
return Class.forName(CORPUS_CLASS);
} catch (ClassNotFoundException exception) {
Assumptions.assumeTrue(false, "Language benchmark corpus is available only when JMH sources are compiled.");
throw exception;
}
}
}

127
src/test/java/org/egothor/stemmer/benchmark/LucenePorterStemmerCopiedTest.java Normal file
View File

@@ -0,0 +1,127 @@
/******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.lang.reflect.Method;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Unit tests for the generated Lucene Porter stemmer benchmark adapter.
*/
@Tag("benchmark")
@Tag("unit")
@DisplayName("LucenePorterStemmerCopied")
final class LucenePorterStemmerCopiedTest {
/**
* Fully qualified benchmark class under test.
*/
private static final String STEMMER_CLASS = "org.egothor.stemmer.benchmark.LucenePorterStemmerCopied";
/**
* Verifies repeated invocations for representative tokens are deterministic.
*/
@Test
@DisplayName("should produce stable stems for representative tokens")
void shouldProduceStableStemsForRepresentativeTokens() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
final String[] tokens = { "running", "caresses", "happiness", "connected", "dancing", "" };
for (String token : tokens) {
final String first = (String) stemMethod.invoke(stemmer, token);
final String second = (String) stemMethod.invoke(stemmer, token);
assertEquals(first, second);
assertNotNull(first);
}
}
/**
* Verifies short tokens are accepted and remain non-null.
*/
@Test
@DisplayName("should return non-null stems for empty and short tokens")
void shouldReturnNonNullStemsForEmptyAndShortTokens() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
assertEquals("", stemMethod.invoke(stemmer, ""));
assertEquals("a", stemMethod.invoke(stemmer, "a"));
assertEquals("go", stemMethod.invoke(stemmer, "go"));
}
/**
* Verifies mutable reuse on a single benchmark instance.
*/
@Test
@DisplayName("should preserve state across many repeated calls on one instance")
void shouldPreserveStateAcrossManyRepeatedCallsOnOneInstance() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
final String first = (String) stemMethod.invoke(stemmer, "connected");
for (int index = 0; index < 64; index++) {
assertEquals(first, stemMethod.invoke(stemmer, "connected"));
}
}
/**
* Instantiates the benchmark copied Lucene Porter class if it is present.
*
* @return benchmark stemmer instance
* @throws Exception when reflective creation fails
*/
private Object createStemmer() throws Exception {
try {
return Class.forName(STEMMER_CLASS).getDeclaredConstructor().newInstance();
} catch (ClassNotFoundException exception) {
Assumptions.assumeTrue(false, "Benchmark Lucene Porter class is available only when JMH sources are compiled.");
throw exception;
}
}
/**
* Resolves the benchmark stem method.
*
* @return `stem` reflection handle
* @throws Exception when reflective lookup fails
*/
private Method stemMethod() throws Exception {
return Class.forName(STEMMER_CLASS).getDeclaredMethod("stem", String.class);
}
}

148
src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java Normal file
View File

@@ -0,0 +1,148 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.lang.reflect.Method;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Unit tests for the benchmark Paice/Husk Lancaster implementation.
*/
@Tag("benchmark")
@Tag("unit")
@DisplayName("PaiceHuskLancasterStemmer")
final class PaiceHuskLancasterStemmerTest {
/**
* Benchmark class under test.
*/
private static final String STEMMER_CLASS = "org.egothor.stemmer.benchmark.PaiceHuskLancasterStemmer";
/**
* Expected outputs used to verify deterministic stems for this benchmark adapter.
*/
private static final String[][] SAMPLE_STEMS = {
{ "running", "run" },
{ "caresses", "cares" },
{ "happiness", "happi" },
{ "connected", "connect" },
{ "dancing", "danc" },
{ "happy", "happy" }
};
/**
* Verifies selected representative words produce stable stems.
*/
@Test
@DisplayName("should produce stable benchmark stems for representative words")
void shouldProduceStableStemsForRepresentativeWords() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
for (String[] sample : SAMPLE_STEMS) {
assertEquals(sample[1], stemMethod.invoke(stemmer, sample[0]));
}
}
/**
* Verifies short and empty inputs remain stable and non-null.
*/
@Test
@DisplayName("should handle empty and short tokens without null output")
void shouldHandleEmptyAndShortTokensWithoutNullOutput() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
assertEquals("", stemMethod.invoke(stemmer, ""));
assertEquals("a", stemMethod.invoke(stemmer, "a"));
assertEquals("x", stemMethod.invoke(stemmer, "x"));
}
/**
* Verifies that one mutable instance can be reused.
*/
@Test
@DisplayName("should be reusable across many repeated calls")
void shouldBeReusableAcrossRepeatedCalls() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
final String first = (String) stemMethod.invoke(stemmer, "running");
final String second = (String) stemMethod.invoke(stemmer, "running");
assertEquals(first, second);
}
/**
* Verifies null input handling for normal integration path expectations.
*/
@Test
@DisplayName("should return null only for null inputs")
void shouldReturnNullOnlyForNullInputs() throws Exception {
final Object stemmer = createStemmer();
final Method stemMethod = stemMethod();
assertEquals("running", stemMethod.invoke(stemmer, "running"));
assertEquals(null, stemMethod.invoke(stemmer, new Object[] { null }));
assertNotNull(stemMethod.invoke(stemmer, "connected"));
}
/**
* Creates a benchmark stemmer instance.
*
* @return benchmark stemmer instance
* @throws Exception when reflection fails
*/
private Object createStemmer() throws Exception {
try {
return Class.forName(STEMMER_CLASS).getDeclaredConstructor().newInstance();
} catch (ClassNotFoundException exception) {
Assumptions.assumeTrue(false, "Benchmark Paice/Husk stemmer is available only when JMH sources are compiled.");
throw exception;
}
}
/**
* Resolves the stem method for benchmark execution.
*
* @return stem method
* @throws Exception when lookup fails
*/
private Method stemMethod() throws Exception {
return Class.forName(STEMMER_CLASS).getDeclaredMethod("stem", String.class);
}
}