Add CISTEM and Hunspell benchmarks and refresh results
This commit is contained in:
178
src/jmh/java/org/egothor/stemmer/benchmark/Cistem.java
Normal file
178
src/jmh/java/org/egothor/stemmer/benchmark/Cistem.java
Normal file
@@ -0,0 +1,178 @@
|
||||
/*******************************************************************************
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2017 Leonie Weißweiler
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*
|
||||
* Source: CISTEM German stemmer
|
||||
* Authors: Leonie Weissweiler, Alexander Fraser
|
||||
* https://github.com/LeonieWeissweiler/CISTEM
|
||||
* https://www.cis.lmu.de/~weissweiler/cistem/
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public final class Cistem {
|
||||
|
||||
private static final Pattern GE_PATTERN = Pattern.compile("^ge(.{4,})");
|
||||
private static final Pattern DOLLAR1_PATTERN = Pattern.compile("(.)\\1");
|
||||
private static final Pattern ND_PATTERN = Pattern.compile("nd$");
|
||||
private static final Pattern EMR_PATTERN = Pattern.compile("e[mr]$");
|
||||
private static final Pattern T_PATTERN = Pattern.compile("t$");
|
||||
private static final Pattern ESN_PATTERN = Pattern.compile("[esn]$");
|
||||
private static final Pattern STAR_PATTERN = Pattern.compile("(.)\\*");
|
||||
|
||||
private Cistem() {
|
||||
}
|
||||
|
||||
public static String stem(final String word) {
|
||||
return stem(word, false);
|
||||
}
|
||||
|
||||
public static String stem(final String word, final boolean caseInsensitive) {
|
||||
if (word.isEmpty()) {
|
||||
return word;
|
||||
}
|
||||
|
||||
String normalized = word;
|
||||
normalized = normalized.replace("Ü", "U");
|
||||
normalized = normalized.replace("Ö", "O");
|
||||
normalized = normalized.replace("Ä", "A");
|
||||
normalized = normalized.replace("ü", "u");
|
||||
normalized = normalized.replace("ö", "o");
|
||||
normalized = normalized.replace("ä", "a");
|
||||
|
||||
final boolean uppercase = Character.isUpperCase(normalized.charAt(0));
|
||||
|
||||
normalized = normalized.toLowerCase();
|
||||
normalized = normalized.replace("ß", "ss");
|
||||
normalized = GE_PATTERN.matcher(normalized).replaceAll("$1");
|
||||
normalized = normalized.replace("sch", "$");
|
||||
normalized = normalized.replace("ei", "%");
|
||||
normalized = normalized.replace("ie", "&");
|
||||
normalized = DOLLAR1_PATTERN.matcher(normalized).replaceAll("$1*");
|
||||
|
||||
while (normalized.length() > 3) {
|
||||
if (normalized.length() > 5) {
|
||||
String newWord = EMR_PATTERN.matcher(normalized).replaceAll("");
|
||||
if (!normalized.equals(newWord)) {
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
|
||||
newWord = ND_PATTERN.matcher(normalized).replaceAll("");
|
||||
if (!normalized.equals(newWord)) {
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!uppercase || caseInsensitive) {
|
||||
final String newWord = T_PATTERN.matcher(normalized).replaceAll("");
|
||||
if (!normalized.equals(newWord)) {
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
final String newWord = ESN_PATTERN.matcher(normalized).replaceAll("");
|
||||
if (!normalized.equals(newWord)) {
|
||||
normalized = newWord;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
normalized = STAR_PATTERN.matcher(normalized).replaceAll("$1$1");
|
||||
normalized = normalized.replace("&", "ie");
|
||||
normalized = normalized.replace("%", "ei");
|
||||
normalized = normalized.replace("$", "sch");
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
public static String[] segment(final String word) {
|
||||
return segment(word, false);
|
||||
}
|
||||
|
||||
public static String[] segment(final String word, final boolean caseInsensitive) {
|
||||
if (word.isEmpty()) {
|
||||
return new String[] {"", ""};
|
||||
}
|
||||
|
||||
int restLength = 0;
|
||||
final boolean uppercase = Character.isUpperCase(word.charAt(0));
|
||||
String normalized = word.toLowerCase();
|
||||
final String original = new String(normalized);
|
||||
|
||||
normalized = normalized.replace("sch", "$");
|
||||
normalized = normalized.replace("ei", "%");
|
||||
normalized = normalized.replace("ie", "&");
|
||||
normalized = DOLLAR1_PATTERN.matcher(normalized).replaceAll("$1*");
|
||||
|
||||
while (normalized.length() > 3) {
|
||||
if (normalized.length() > 5) {
|
||||
String newWord = normalized.replaceAll("e[mr]$", "");
|
||||
if (!normalized.equals(newWord)) {
|
||||
restLength += 2;
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
|
||||
newWord = normalized.replaceAll("nd$", "");
|
||||
if (!normalized.equals(newWord)) {
|
||||
restLength += 2;
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!uppercase || caseInsensitive) {
|
||||
final String newWord = normalized.replaceAll("t$", "");
|
||||
if (!normalized.equals(newWord)) {
|
||||
restLength += 1;
|
||||
normalized = newWord;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
final String newWord = normalized.replaceAll("[esn]$", "");
|
||||
if (!normalized.equals(newWord)) {
|
||||
restLength += 1;
|
||||
normalized = newWord;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
normalized = normalized.replaceAll("(.)\\*", "$1$1");
|
||||
normalized = normalized.replace("&", "ie");
|
||||
normalized = normalized.replace("%", "ei");
|
||||
normalized = normalized.replace("$", "sch");
|
||||
|
||||
String rest = "";
|
||||
if (restLength != 0) {
|
||||
rest = original.substring(original.length() - restLength);
|
||||
}
|
||||
|
||||
return new String[] {normalized, rest};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.lucene.analysis.LowerCaseFilter;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.de.GermanLightStemFilter;
|
||||
import org.apache.lucene.analysis.de.GermanMinimalStemFilter;
|
||||
import org.apache.lucene.analysis.de.GermanNormalizationFilter;
|
||||
import org.apache.lucene.analysis.de.GermanStemFilter;
|
||||
import org.apache.lucene.analysis.snowball.SnowballFilter;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
|
||||
import org.openjdk.jmh.annotations.AuxCounters;
|
||||
import org.openjdk.jmh.annotations.Benchmark;
|
||||
import org.openjdk.jmh.annotations.BenchmarkMode;
|
||||
import org.openjdk.jmh.annotations.Fork;
|
||||
import org.openjdk.jmh.annotations.Level;
|
||||
import org.openjdk.jmh.annotations.Measurement;
|
||||
import org.openjdk.jmh.annotations.Mode;
|
||||
import org.openjdk.jmh.annotations.OutputTimeUnit;
|
||||
import org.openjdk.jmh.annotations.Param;
|
||||
import org.openjdk.jmh.annotations.Scope;
|
||||
import org.openjdk.jmh.annotations.Setup;
|
||||
import org.openjdk.jmh.annotations.State;
|
||||
import org.openjdk.jmh.annotations.Warmup;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.ReductionMode;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
/**
|
||||
* German-only stemmer comparison on CISTEM gold standards.
|
||||
*
|
||||
* <p>
|
||||
* Each benchmark operation is fed by one cluster file. The same candidate set is
|
||||
* evaluated twice, once per file, to produce one precision/recall/f-measure
|
||||
* table for each gold standard.
|
||||
* </p>
|
||||
*/
|
||||
@BenchmarkMode(Mode.AverageTime)
|
||||
@OutputTimeUnit(java.util.concurrent.TimeUnit.NANOSECONDS)
|
||||
@Warmup(iterations = 3, time = 1, timeUnit = java.util.concurrent.TimeUnit.SECONDS)
|
||||
@Measurement(iterations = 5, time = 1, timeUnit = java.util.concurrent.TimeUnit.SECONDS)
|
||||
@Fork(1)
|
||||
public class GermanGoldstandardStemmerComparisonBenchmark {
|
||||
|
||||
/**
|
||||
* Shared German benchmark state for one dataset and one candidate.
|
||||
*/
|
||||
@State(Scope.Benchmark)
|
||||
public static class SharedState {
|
||||
|
||||
/**
|
||||
* Gold standard dataset.
|
||||
*/
|
||||
@Param({"goldstandard1.txt", "goldstandard2.txt"})
|
||||
public String goldStandardFileName;
|
||||
|
||||
/**
|
||||
* Candidate stemmer.
|
||||
*/
|
||||
@Param({
|
||||
"GERMAN_RADIXOR",
|
||||
"GERMAN_LUCENE_GERMAN_STEM_FILTER",
|
||||
"GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER",
|
||||
"GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER",
|
||||
"GERMAN_CISTEM",
|
||||
"SNOWBALL_GERMAN_DIRECT",
|
||||
"SNOWBALL_GERMAN_LUCENE_FILTER"
|
||||
})
|
||||
public String candidateName;
|
||||
|
||||
/**
|
||||
* Parsed gold standard corpus.
|
||||
*/
|
||||
private GermanGoldstandardCorpus corpus;
|
||||
|
||||
/**
|
||||
* Gold standard words flattened by cluster order.
|
||||
*/
|
||||
private String[] allTokens;
|
||||
|
||||
/**
|
||||
* Candidate evaluator.
|
||||
*/
|
||||
private GoldstandardStemmer stemmer;
|
||||
|
||||
/**
|
||||
* Initializes one candidate on one gold standard corpus.
|
||||
*
|
||||
* @throws IOException when the corpus cannot be loaded
|
||||
*/
|
||||
@Setup(Level.Trial)
|
||||
public void setUp() throws IOException {
|
||||
this.corpus = loadCorpus(this.goldStandardFileName);
|
||||
this.allTokens = flattenCorpusTokens(this.corpus);
|
||||
this.stemmer = GermanCandidate.valueOf(this.candidateName).createEvaluator();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* JMH auxiliary counters for CISTEM-style cluster accounting.
|
||||
*/
|
||||
@State(Scope.Thread)
|
||||
@AuxCounters(AuxCounters.Type.EVENTS)
|
||||
public static class GoldstandardQualityCounters {
|
||||
|
||||
/**
|
||||
* True positives across clusters.
|
||||
*/
|
||||
public long truePositives;
|
||||
|
||||
/**
|
||||
* False positives across clusters.
|
||||
*/
|
||||
public long falsePositives;
|
||||
|
||||
/**
|
||||
* False negatives across clusters.
|
||||
*/
|
||||
public long falseNegatives;
|
||||
|
||||
/**
|
||||
* Evaluated clusters.
|
||||
*/
|
||||
public long evaluatedClusters;
|
||||
|
||||
/**
|
||||
* Evaluated tokens.
|
||||
*/
|
||||
public long evaluatedTokens;
|
||||
|
||||
/**
|
||||
* Resets counters before each measured iteration.
|
||||
*/
|
||||
@Setup(Level.Iteration)
|
||||
public void reset() {
|
||||
this.truePositives = 0L;
|
||||
this.falsePositives = 0L;
|
||||
this.falseNegatives = 0L;
|
||||
this.evaluatedClusters = 0L;
|
||||
this.evaluatedTokens = 0L;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates CISTEM-style precision, recall, and F1-relevant counts.
|
||||
*
|
||||
* @param state shared benchmark state
|
||||
* @param counters quality counters
|
||||
* @param blackhole result sink
|
||||
* @return evaluated token count for this operation
|
||||
* @throws IOException if token filtering cannot run
|
||||
*/
|
||||
@Benchmark
|
||||
@Warmup(iterations = 0)
|
||||
@Measurement(iterations = 1, time = 1, timeUnit = java.util.concurrent.TimeUnit.MILLISECONDS)
|
||||
@Fork(0)
|
||||
public long cistemStyleQuality(final SharedState state, final GoldstandardQualityCounters counters,
|
||||
final Blackhole blackhole) throws IOException {
|
||||
final GoldstandardResult result = evaluateCistemStyle(state.corpus, state.allTokens, state.stemmer, blackhole);
|
||||
counters.truePositives += result.truePositives();
|
||||
counters.falsePositives += result.falsePositives();
|
||||
counters.falseNegatives += result.falseNegatives();
|
||||
counters.evaluatedClusters += result.evaluatedClusters();
|
||||
counters.evaluatedTokens += result.evaluatedTokens();
|
||||
return result.evaluatedTokens();
|
||||
}
|
||||
|
||||
/**
|
||||
* Benchmarks candidate throughput over the selected gold standard.
|
||||
*
|
||||
* @param state shared benchmark state
|
||||
* @param blackhole result sink
|
||||
* @throws IOException if token filtering cannot run
|
||||
*/
|
||||
@Benchmark
|
||||
public void cistemStyleSpeed(final SharedState state, final Blackhole blackhole) throws IOException {
|
||||
state.stemmer.stem(state.allTokens, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Named German candidates used for the CISTEM gold-standard comparison.
|
||||
*/
|
||||
private enum GermanCandidate {
|
||||
GERMAN_RADIXOR,
|
||||
GERMAN_LUCENE_GERMAN_STEM_FILTER,
|
||||
GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER,
|
||||
GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER,
|
||||
GERMAN_CISTEM,
|
||||
SNOWBALL_GERMAN_DIRECT,
|
||||
SNOWBALL_GERMAN_LUCENE_FILTER;
|
||||
|
||||
/**
|
||||
* Creates a candidate evaluator.
|
||||
*
|
||||
* @return stemmer evaluator
|
||||
* @throws IOException if trie resources cannot be loaded
|
||||
*/
|
||||
GoldstandardStemmer createEvaluator() throws IOException {
|
||||
return switch (this) {
|
||||
case GERMAN_RADIXOR -> direct(createGermanRadixorStemmer());
|
||||
case GERMAN_LUCENE_GERMAN_STEM_FILTER ->
|
||||
tokenFilter(input -> new GermanStemFilter(lowercase(input)));
|
||||
case GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER ->
|
||||
tokenFilter(input -> new GermanLightStemFilter(germanNormalize(input)));
|
||||
case GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER ->
|
||||
tokenFilter(input -> new GermanMinimalStemFilter(germanNormalize(input)));
|
||||
case GERMAN_CISTEM -> direct(Cistem::stem);
|
||||
case SNOWBALL_GERMAN_DIRECT -> direct(SnowballLanguageCase.GERMAN.createDirectStemmer()::stem);
|
||||
case SNOWBALL_GERMAN_LUCENE_FILTER ->
|
||||
tokenFilter(input -> new SnowballFilter(new LowerCaseFilter(input),
|
||||
SnowballLanguageCase.GERMAN.luceneSnowballName()));
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates one full corpus through CISTEM-style cluster scoring.
|
||||
*
|
||||
* <p>
|
||||
* For each cluster, the most frequent predicted stem is considered the
|
||||
* cluster main stem. TP are cluster words mapped to this stem, FN are
|
||||
* words mapped elsewhere inside the same cluster, and FP are words from
|
||||
* other clusters mapped to the same main stem.
|
||||
* </p>
|
||||
*
|
||||
* @param corpus parsed gold standard corpus
|
||||
* @param allTokens flattened token sequence
|
||||
* @param stemmer candidate stemmer
|
||||
* @param blackhole result sink
|
||||
* @return aggregated TP/FP/FN counters and token metrics
|
||||
* @throws IOException when token filtering cannot run
|
||||
*/
|
||||
private static GoldstandardResult evaluateCistemStyle(final GermanGoldstandardCorpus corpus,
|
||||
final String[] allTokens, final GoldstandardStemmer stemmer, final Blackhole blackhole) throws IOException {
|
||||
final String[] predicted = stemmer.stem(allTokens, blackhole);
|
||||
final Map<String, Integer> globalPredictions = new LinkedHashMap<>();
|
||||
for (int index = 0; index < allTokens.length; index++) {
|
||||
final String prediction = normalizePrediction(predicted[index], allTokens[index]);
|
||||
globalPredictions.put(prediction, globalPredictions.getOrDefault(prediction, 0) + 1);
|
||||
}
|
||||
|
||||
long truePositives = 0L;
|
||||
long falsePositives = 0L;
|
||||
long falseNegatives = 0L;
|
||||
int tokenOffset = 0;
|
||||
for (final String[] cluster : corpus.clusters()) {
|
||||
if (cluster.length == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final Map<String, Integer> localPredictions = new LinkedHashMap<>();
|
||||
for (int index = 0; index < cluster.length; index++) {
|
||||
final int tokenIndex = tokenOffset + index;
|
||||
final String word = allTokens[tokenIndex];
|
||||
final String prediction = normalizePrediction(predicted[tokenIndex], word);
|
||||
localPredictions.put(prediction, localPredictions.getOrDefault(prediction, 0) + 1);
|
||||
}
|
||||
|
||||
final String mainStem = mostFrequent(localPredictions);
|
||||
final int predictedAsMain = localPredictions.get(mainStem);
|
||||
final int clusterSize = cluster.length;
|
||||
final int falseNegative = clusterSize - predictedAsMain;
|
||||
final int falsePositive = globalPredictions.get(mainStem) - predictedAsMain;
|
||||
|
||||
truePositives += predictedAsMain;
|
||||
falseNegatives += falseNegative;
|
||||
falsePositives += falsePositive;
|
||||
tokenOffset += clusterSize;
|
||||
}
|
||||
|
||||
return new GoldstandardResult(truePositives, falsePositives, falseNegatives, corpus.clusters().length,
|
||||
allTokens.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a direct evaluator.
|
||||
*
|
||||
* @param stemmer direct word stemmer
|
||||
* @return evaluator
|
||||
*/
|
||||
private static GoldstandardStemmer direct(final Stemmer stemmer) {
|
||||
Objects.requireNonNull(stemmer, "stemmer");
|
||||
return (tokens, blackhole) -> {
|
||||
final String[] outputs = new String[tokens.length];
|
||||
for (int index = 0; index < tokens.length; index++) {
|
||||
final String output = stemmer.stem(tokens[index]);
|
||||
outputs[index] = output;
|
||||
blackhole.consume(output);
|
||||
}
|
||||
return outputs;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a TokenFilter evaluator.
|
||||
*
|
||||
* @param factory filter stream factory
|
||||
* @return evaluator
|
||||
*/
|
||||
private static GoldstandardStemmer tokenFilter(final Function<TokenStream, TokenStream> factory) {
|
||||
Objects.requireNonNull(factory, "factory");
|
||||
return (tokens, blackhole) -> firstTokenFilterOutputs(tokens, factory, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads and parses one gold standard file from generated JMH resources.
|
||||
*
|
||||
* @param resourceName gold standard file name
|
||||
* @return parsed corpus
|
||||
* @throws IOException if reading fails
|
||||
*/
|
||||
private static GermanGoldstandardCorpus loadCorpus(final String resourceName) throws IOException {
|
||||
final ClassLoader classLoader = GermanGoldstandardStemmerComparisonBenchmark.class.getClassLoader();
|
||||
final InputStream resourceStream = classLoader.getResourceAsStream(resourceName);
|
||||
if (resourceStream == null) {
|
||||
throw new IllegalStateException("Missing generated CISTEM gold standard resource: " + resourceName
|
||||
+ ". Run the Gradle JMH resource preparation task to download benchmark-only inputs.");
|
||||
}
|
||||
try (InputStream input = resourceStream) {
|
||||
return parseCorpus(new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses CISTEM gold standard format into clustered candidates.
|
||||
*
|
||||
* @param reader UTF-8 reader
|
||||
* @return parsed corpus
|
||||
* @throws IOException if input cannot be read
|
||||
*/
|
||||
private static GermanGoldstandardCorpus parseCorpus(final BufferedReader reader) throws IOException {
|
||||
final List<String[]> clusters = new ArrayList<>();
|
||||
String line = reader.readLine();
|
||||
while (line != null) {
|
||||
final String trimmed = line.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
final String[] words = trimmed.split("\\s+");
|
||||
if (words.length > 0) {
|
||||
clusters.add(words);
|
||||
}
|
||||
}
|
||||
line = reader.readLine();
|
||||
}
|
||||
return new GermanGoldstandardCorpus(clusters.toArray(String[][]::new));
|
||||
}
|
||||
|
||||
/**
|
||||
* Flattens the corpus in deterministic cluster order.
|
||||
*
|
||||
* @param corpus corpus to flatten
|
||||
* @return flattened token array
|
||||
*/
|
||||
private static String[] flattenCorpusTokens(final GermanGoldstandardCorpus corpus) {
|
||||
int total = 0;
|
||||
for (final String[] cluster : corpus.clusters()) {
|
||||
total += cluster.length;
|
||||
}
|
||||
final String[] tokens = new String[total];
|
||||
int index = 0;
|
||||
for (final String[] cluster : corpus.clusters()) {
|
||||
System.arraycopy(cluster, 0, tokens, index, cluster.length);
|
||||
index += cluster.length;
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the most frequent key; insertion order is preserved on ties.
|
||||
*
|
||||
* @param frequencies predicted stem frequencies
|
||||
* @return most frequent stem
|
||||
*/
|
||||
private static String mostFrequent(final Map<String, Integer> frequencies) {
|
||||
String best = null;
|
||||
int bestCount = -1;
|
||||
for (final Map.Entry<String, Integer> entry : frequencies.entrySet()) {
|
||||
if (entry.getValue() > bestCount) {
|
||||
best = entry.getKey();
|
||||
bestCount = entry.getValue();
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a null/empty prediction using the input token as fallback.
|
||||
*
|
||||
* @param prediction stemmed token
|
||||
* @param fallback fallback token
|
||||
* @return safe prediction
|
||||
*/
|
||||
private static String normalizePrediction(final String prediction, final String fallback) {
|
||||
if (prediction == null || prediction.isEmpty()) {
|
||||
return fallback;
|
||||
}
|
||||
return prediction;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies one TokenFilter to all input tokens and returns the first emitted term
|
||||
* for each input token.
|
||||
*
|
||||
* @param tokens input token corpus
|
||||
* @param factory TokenFilter factory
|
||||
* @param blackhole result sink
|
||||
* @return first emitted term per input token
|
||||
* @throws IOException if token streaming fails
|
||||
*/
|
||||
private static String[] firstTokenFilterOutputs(final String[] tokens, final Function<TokenStream, TokenStream> factory,
|
||||
final Blackhole blackhole) throws IOException {
|
||||
final String[] outputs = new String[tokens.length];
|
||||
final BenchmarkTokenStream input = new BenchmarkTokenStream(tokens);
|
||||
final TokenStream output = factory.apply(input);
|
||||
final CharTermAttribute termAttribute = output.addAttribute(CharTermAttribute.class);
|
||||
final PositionIncrementAttribute positionAttribute = output.addAttribute(PositionIncrementAttribute.class);
|
||||
|
||||
int inputIndex = -1;
|
||||
boolean recordedForPosition = false;
|
||||
output.reset();
|
||||
while (output.incrementToken()) {
|
||||
final int positionIncrement = positionAttribute.getPositionIncrement();
|
||||
if (positionIncrement > 0) {
|
||||
inputIndex += positionIncrement;
|
||||
recordedForPosition = false;
|
||||
}
|
||||
if (inputIndex >= 0 && inputIndex < outputs.length && !recordedForPosition) {
|
||||
outputs[inputIndex] = termAttribute.toString();
|
||||
blackhole.consume(termAttribute);
|
||||
recordedForPosition = true;
|
||||
}
|
||||
}
|
||||
output.end();
|
||||
output.close();
|
||||
|
||||
for (int index = 0; index < outputs.length; index++) {
|
||||
if (outputs[index] == null) {
|
||||
outputs[index] = tokens[index];
|
||||
}
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a direct Radixor evaluator using the contracted dictionary trie.
|
||||
*
|
||||
* @return direct Radixor stemmer
|
||||
* @throws IOException if the trie cannot be loaded
|
||||
*/
|
||||
private static Stemmer createGermanRadixorStemmer() throws IOException {
|
||||
return new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
|
||||
StemmerPatchTrieLoader.Language.DE_DE, true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS))::stem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds Lucene lower-case normalization.
|
||||
*
|
||||
* @param input token stream
|
||||
* @return normalized token stream
|
||||
*/
|
||||
private static TokenStream lowercase(final TokenStream input) {
|
||||
return new LowerCaseFilter(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds Lucene German normalization for light and minimal filters.
|
||||
*
|
||||
* @param input token stream
|
||||
* @return normalized token stream
|
||||
*/
|
||||
private static TokenStream germanNormalize(final TokenStream input) {
|
||||
return new GermanNormalizationFilter(lowercase(input));
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct or filter stemmer adapter used by this benchmark.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
private interface GoldstandardStemmer {
|
||||
|
||||
/**
|
||||
* Runs one complete token list.
|
||||
*
|
||||
* @param tokens input tokens
|
||||
* @param blackhole result sink
|
||||
* @return per-token outputs
|
||||
* @throws IOException if filter processing fails
|
||||
*/
|
||||
String[] stem(String[] tokens, Blackhole blackhole) throws IOException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic direct word stem function.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
private interface Stemmer {
|
||||
|
||||
/**
|
||||
* Stems one token.
|
||||
*
|
||||
* @param token input token
|
||||
* @return stemmed token
|
||||
*/
|
||||
String stem(String token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Immutable parsed CISTEM gold standard corpus.
|
||||
*/
|
||||
private static final class GermanGoldstandardCorpus {
|
||||
|
||||
private final String[][] clusters;
|
||||
|
||||
GermanGoldstandardCorpus(final String[][] clusters) {
|
||||
this.clusters = clusters;
|
||||
}
|
||||
|
||||
String[][] clusters() {
|
||||
return this.clusters;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregated quality result for one benchmark operation.
|
||||
*
|
||||
* @param truePositives true positives
|
||||
* @param falsePositives false positives
|
||||
* @param falseNegatives false negatives
|
||||
* @param evaluatedClusters evaluated clusters
|
||||
* @param evaluatedTokens evaluated tokens
|
||||
*/
|
||||
private record GoldstandardResult(long truePositives, long falsePositives, long falseNegatives,
|
||||
long evaluatedClusters, long evaluatedTokens) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.text.ParseException;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.lucene.analysis.LowerCaseFilter;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.hunspell.Dictionary;
|
||||
import org.apache.lucene.analysis.hunspell.HunspellStemFilter;
|
||||
import org.apache.lucene.analysis.hunspell.SortingStrategy;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
import org.openjdk.jmh.annotations.AuxCounters;
|
||||
import org.openjdk.jmh.annotations.Benchmark;
|
||||
import org.openjdk.jmh.annotations.BenchmarkMode;
|
||||
import org.openjdk.jmh.annotations.Fork;
|
||||
import org.openjdk.jmh.annotations.Level;
|
||||
import org.openjdk.jmh.annotations.Measurement;
|
||||
import org.openjdk.jmh.annotations.Mode;
|
||||
import org.openjdk.jmh.annotations.OutputTimeUnit;
|
||||
import org.openjdk.jmh.annotations.Param;
|
||||
import org.openjdk.jmh.annotations.Scope;
|
||||
import org.openjdk.jmh.annotations.Setup;
|
||||
import org.openjdk.jmh.annotations.State;
|
||||
import org.openjdk.jmh.annotations.Warmup;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
/**
|
||||
* Emits exact-root agreement metrics for the benchmark-only Hunspell comparisons.
|
||||
*
|
||||
* <p>
|
||||
* This class mirrors the existing Hunspell throughput setup but adds
|
||||
* quality-style accuracy counters for every Hunspell language dictionary used
|
||||
* in benchmark-only throughput comparisons.
|
||||
* </p>
|
||||
*/
|
||||
@BenchmarkMode(Mode.AverageTime)
|
||||
@OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
@Warmup(iterations = 0)
|
||||
@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS)
|
||||
@Fork(0)
|
||||
public class HunspellStemmerComparisonBenchmarkQuality {
|
||||
|
||||
/**
|
||||
* Shared quality corpus and Hunspell dictionary for a selected language.
|
||||
*/
|
||||
@State(Scope.Benchmark)
|
||||
public static class SharedState {
|
||||
|
||||
/**
|
||||
* Selected language case.
|
||||
*/
|
||||
@Param({ "ENGLISH", "CZECH", "GERMAN", "SPANISH", "FRENCH", "DUTCH", "POLISH", "UKRAINIAN" })
|
||||
public String languageCaseName;
|
||||
|
||||
/**
|
||||
* Selected language descriptor.
|
||||
*/
|
||||
private HunspellLanguageCase languageCase;
|
||||
|
||||
/**
|
||||
* Complete language dictionary corpus and expected roots.
|
||||
*/
|
||||
private LanguageBenchmarkCorpus.Corpus corpus;
|
||||
|
||||
/**
|
||||
* Parsed benchmark-only Hunspell dictionary.
|
||||
*/
|
||||
private Dictionary dictionary;
|
||||
|
||||
/**
|
||||
* Initializes quality resources.
|
||||
*
|
||||
* @throws IOException if corpus or dictionary loading fails
|
||||
* @throws ParseException if the Hunspell dictionary cannot be parsed
|
||||
*/
|
||||
@Setup(Level.Trial)
|
||||
public void setUp() throws IOException, ParseException {
|
||||
this.languageCase = HunspellLanguageCase.valueOf(this.languageCaseName);
|
||||
this.corpus = LanguageBenchmarkCorpus.createFullCorpus(this.languageCase.radixorLanguage());
|
||||
this.dictionary = loadDictionary(this.languageCase);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* JMH auxiliary counters for exact-root agreement.
|
||||
*/
|
||||
@State(Scope.Thread)
|
||||
@AuxCounters(AuxCounters.Type.EVENTS)
|
||||
public static class AccuracyCounters {
|
||||
|
||||
/**
|
||||
* Number of exact-root matches.
|
||||
*/
|
||||
public long correctMatches;
|
||||
|
||||
/**
|
||||
* Number of evaluated tokens.
|
||||
*/
|
||||
public long evaluatedTokens;
|
||||
|
||||
/**
|
||||
* Number of exact-root matches where the input token differs from the
|
||||
* expected root.
|
||||
*/
|
||||
public long changedCorrectMatches;
|
||||
|
||||
/**
|
||||
* Number of evaluated tokens where the input token differs from the expected
|
||||
* root.
|
||||
*/
|
||||
public long changedEvaluatedTokens;
|
||||
|
||||
/**
|
||||
* Number of exact-root matches where the input token is already the expected
|
||||
* root.
|
||||
*/
|
||||
public long rootPreservedMatches;
|
||||
|
||||
/**
|
||||
* Number of evaluated tokens where the input token is already the expected
|
||||
* root.
|
||||
*/
|
||||
public long rootEvaluatedTokens;
|
||||
|
||||
/**
|
||||
* Resets counters before each measured iteration.
|
||||
*/
|
||||
@Setup(Level.Iteration)
|
||||
public void reset() {
|
||||
this.correctMatches = 0L;
|
||||
this.evaluatedTokens = 0L;
|
||||
this.changedCorrectMatches = 0L;
|
||||
this.changedEvaluatedTokens = 0L;
|
||||
this.rootPreservedMatches = 0L;
|
||||
this.rootEvaluatedTokens = 0L;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates exact-root agreement for the selected Hunspell dictionary.
|
||||
*
|
||||
* @param sharedState shared quality state
|
||||
* @param counters JMH auxiliary counters
|
||||
* @param blackhole result sink
|
||||
* @return exact-root match count
|
||||
* @throws IOException if Lucene token streaming fails
|
||||
*/
|
||||
@Benchmark
|
||||
public int luceneHunspellStemFilterAccuracy(final SharedState sharedState, final AccuracyCounters counters,
|
||||
final Blackhole blackhole) throws IOException {
|
||||
final String[] actualStems = firstHunspellOutputs(sharedState.corpus.tokens(), sharedState.dictionary,
|
||||
blackhole);
|
||||
final String[] tokens = sharedState.corpus.tokens();
|
||||
final String[] expectedRoots = sharedState.corpus.expectedRoots();
|
||||
|
||||
int correct = 0;
|
||||
int changedCorrect = 0;
|
||||
int changedEvaluated = 0;
|
||||
int rootPreserved = 0;
|
||||
int rootEvaluated = 0;
|
||||
for (int index = 0; index < actualStems.length; index++) {
|
||||
final String token = tokens[index];
|
||||
final String expectedRoot = expectedRoots[index];
|
||||
final boolean exact = Objects.equals(expectedRoot, actualStems[index]);
|
||||
if (exact) {
|
||||
correct++;
|
||||
}
|
||||
if (Objects.equals(token, expectedRoot)) {
|
||||
rootEvaluated++;
|
||||
if (exact) {
|
||||
rootPreserved++;
|
||||
}
|
||||
} else {
|
||||
changedEvaluated++;
|
||||
if (exact) {
|
||||
changedCorrect++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
counters.correctMatches += correct;
|
||||
counters.evaluatedTokens += actualStems.length;
|
||||
counters.changedCorrectMatches += changedCorrect;
|
||||
counters.changedEvaluatedTokens += changedEvaluated;
|
||||
counters.rootPreservedMatches += rootPreserved;
|
||||
counters.rootEvaluatedTokens += rootEvaluated;
|
||||
return correct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the first emitted Hunspell stem for each input token.
|
||||
*
|
||||
* @param tokens token corpus
|
||||
* @param dictionary Hunspell dictionary
|
||||
* @param blackhole result sink
|
||||
* @return first emitted term per input token
|
||||
* @throws IOException if Lucene streaming fails
|
||||
*/
|
||||
private static String[] firstHunspellOutputs(final String[] tokens, final Dictionary dictionary,
|
||||
final Blackhole blackhole) throws IOException {
|
||||
final String[] outputs = new String[tokens.length];
|
||||
final BenchmarkTokenStream input = new BenchmarkTokenStream(tokens);
|
||||
final TokenStream output = new HunspellStemFilter(new LowerCaseFilter(input), dictionary, true);
|
||||
final CharTermAttribute termAttribute = output.addAttribute(CharTermAttribute.class);
|
||||
final PositionIncrementAttribute positionAttribute = output.addAttribute(PositionIncrementAttribute.class);
|
||||
int inputIndex = -1;
|
||||
boolean recordedForPosition = false;
|
||||
|
||||
output.reset();
|
||||
while (output.incrementToken()) {
|
||||
final int positionIncrement = positionAttribute.getPositionIncrement();
|
||||
if (positionIncrement > 0) {
|
||||
inputIndex += positionIncrement;
|
||||
recordedForPosition = false;
|
||||
}
|
||||
if (inputIndex >= 0 && inputIndex < outputs.length && !recordedForPosition) {
|
||||
outputs[inputIndex] = termAttribute.toString();
|
||||
recordedForPosition = true;
|
||||
}
|
||||
blackhole.consume(termAttribute);
|
||||
}
|
||||
output.end();
|
||||
output.close();
|
||||
|
||||
for (int index = 0; index < outputs.length; index++) {
|
||||
if (outputs[index] == null) {
|
||||
outputs[index] = tokens[index];
|
||||
}
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a benchmark-only Hunspell dictionary from generated resources.
|
||||
*
|
||||
* @param languageCase selected language case
|
||||
* @return parsed dictionary
|
||||
* @throws IOException if dictionary resources cannot be read
|
||||
* @throws ParseException if dictionary parsing fails
|
||||
*/
|
||||
private static Dictionary loadDictionary(final HunspellLanguageCase languageCase) throws IOException,
|
||||
ParseException {
|
||||
final ClassLoader classLoader = HunspellStemmerComparisonBenchmarkQuality.class.getClassLoader();
|
||||
final String basePath = "hunspell/" + languageCase.hunspellResourceCode() + "/index.";
|
||||
try (InputStream affixStream = openRequiredResource(classLoader, basePath + "aff");
|
||||
InputStream dictionaryStream = openRequiredResource(classLoader, basePath + "dic")) {
|
||||
return new Dictionary(affixStream, List.of(dictionaryStream), true, SortingStrategy.inMemory());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a required classpath resource.
|
||||
*
|
||||
* @param classLoader class loader
|
||||
* @param path resource path
|
||||
* @return resource stream
|
||||
*/
|
||||
private static InputStream openRequiredResource(final ClassLoader classLoader, final String path) {
|
||||
final InputStream stream = classLoader.getResourceAsStream(path);
|
||||
if (stream == null) {
|
||||
throw new IllegalStateException("Missing benchmark-only Hunspell resource: " + path);
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Benchmark language mapping.
|
||||
*/
|
||||
private enum HunspellLanguageCase {
|
||||
|
||||
/**
|
||||
* English Hunspell dictionary over the Radixor English corpus.
|
||||
*/
|
||||
ENGLISH("en", StemmerPatchTrieLoader.Language.US_UK),
|
||||
|
||||
/**
|
||||
* Czech Hunspell dictionary over the Radixor Czech corpus.
|
||||
*/
|
||||
CZECH("cs", StemmerPatchTrieLoader.Language.CS_CZ),
|
||||
|
||||
/**
|
||||
* German Hunspell dictionary over the Radixor German corpus.
|
||||
*/
|
||||
GERMAN("de", StemmerPatchTrieLoader.Language.DE_DE),
|
||||
|
||||
/**
|
||||
* Spanish Hunspell dictionary over the Radixor Spanish corpus.
|
||||
*/
|
||||
SPANISH("es", StemmerPatchTrieLoader.Language.ES_ES),
|
||||
|
||||
/**
|
||||
* French Hunspell dictionary over the Radixor French corpus.
|
||||
*/
|
||||
FRENCH("fr", StemmerPatchTrieLoader.Language.FR_FR),
|
||||
|
||||
/**
|
||||
* Dutch Hunspell dictionary over the Radixor Dutch corpus.
|
||||
*/
|
||||
DUTCH("nl", StemmerPatchTrieLoader.Language.NL_NL),
|
||||
|
||||
/**
|
||||
* Polish Hunspell dictionary over the Radixor Polish corpus.
|
||||
*/
|
||||
POLISH("pl", StemmerPatchTrieLoader.Language.PL_PL),
|
||||
|
||||
/**
|
||||
* Ukrainian Hunspell dictionary over the Radixor Ukrainian corpus.
|
||||
*/
|
||||
UKRAINIAN("uk", StemmerPatchTrieLoader.Language.UK_UA);
|
||||
|
||||
/**
|
||||
* Wooorm/dictionaries resource code.
|
||||
*/
|
||||
private final String hunspellResourceCode;
|
||||
|
||||
/**
|
||||
* Matching Radixor language.
|
||||
*/
|
||||
private final StemmerPatchTrieLoader.Language radixorLanguage;
|
||||
|
||||
/**
|
||||
* Creates a language mapping.
|
||||
*
|
||||
* @param hunspellResourceCode Hunspell resource code
|
||||
* @param radixorLanguage Radixor language
|
||||
*/
|
||||
HunspellLanguageCase(final String hunspellResourceCode, final StemmerPatchTrieLoader.Language radixorLanguage) {
|
||||
this.hunspellResourceCode = hunspellResourceCode.toLowerCase(Locale.ROOT);
|
||||
this.radixorLanguage = radixorLanguage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Hunspell resource code.
|
||||
*
|
||||
* @return resource code
|
||||
*/
|
||||
String hunspellResourceCode() {
|
||||
return this.hunspellResourceCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the matching Radixor language.
|
||||
*
|
||||
* @return Radixor language
|
||||
*/
|
||||
StemmerPatchTrieLoader.Language radixorLanguage() {
|
||||
return this.radixorLanguage;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -473,6 +473,19 @@ public class MultiLanguageStemmerComparisonBenchmark {
|
||||
filterState.germanMinimalStem.run(sharedState.german.tokens, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs CISTEM directly over the German corpus.
|
||||
*
|
||||
* @param sharedState shared benchmark state
|
||||
* @param blackhole result sink
|
||||
*/
|
||||
@Benchmark
|
||||
public void germanCistem(final SharedState sharedState, final Blackhole blackhole) {
|
||||
for (final String token : sharedState.german.tokens) {
|
||||
blackhole.consume(Cistem.stem(token));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs Radixor over the Spanish corpus.
|
||||
*
|
||||
|
||||
@@ -144,6 +144,7 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
"GERMAN_LUCENE_GERMAN_STEM_FILTER",
|
||||
"GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER",
|
||||
"GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER",
|
||||
"GERMAN_CISTEM",
|
||||
"SPANISH_RADIXOR",
|
||||
"SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER",
|
||||
"SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER",
|
||||
@@ -178,46 +179,32 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
"UKRAINIAN_RADIXOR",
|
||||
"UKRAINIAN_MORFOLOGIK_DIRECT",
|
||||
"UKRAINIAN_LUCENE_MORFOLOGIK_FILTER",
|
||||
"SNOWBALL_DANISH_RADIXOR",
|
||||
"SNOWBALL_DANISH_DIRECT",
|
||||
"SNOWBALL_DANISH_LUCENE_FILTER",
|
||||
"SNOWBALL_DUTCH_RADIXOR",
|
||||
"SNOWBALL_DUTCH_DIRECT",
|
||||
"SNOWBALL_DUTCH_LUCENE_FILTER",
|
||||
"SNOWBALL_FINNISH_RADIXOR",
|
||||
"SNOWBALL_FINNISH_DIRECT",
|
||||
"SNOWBALL_FINNISH_LUCENE_FILTER",
|
||||
"SNOWBALL_FRENCH_RADIXOR",
|
||||
"SNOWBALL_FRENCH_DIRECT",
|
||||
"SNOWBALL_FRENCH_LUCENE_FILTER",
|
||||
"SNOWBALL_GERMAN_RADIXOR",
|
||||
"SNOWBALL_GERMAN_DIRECT",
|
||||
"SNOWBALL_GERMAN_LUCENE_FILTER",
|
||||
"SNOWBALL_HUNGARIAN_RADIXOR",
|
||||
"SNOWBALL_HUNGARIAN_DIRECT",
|
||||
"SNOWBALL_HUNGARIAN_LUCENE_FILTER",
|
||||
"SNOWBALL_ITALIAN_RADIXOR",
|
||||
"SNOWBALL_ITALIAN_DIRECT",
|
||||
"SNOWBALL_ITALIAN_LUCENE_FILTER",
|
||||
"SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR",
|
||||
"SNOWBALL_NORWEGIAN_BOKMAL_DIRECT",
|
||||
"SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER",
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR",
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_DIRECT",
|
||||
"SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER",
|
||||
"SNOWBALL_PORTUGUESE_RADIXOR",
|
||||
"SNOWBALL_PORTUGUESE_DIRECT",
|
||||
"SNOWBALL_PORTUGUESE_LUCENE_FILTER",
|
||||
"SNOWBALL_RUSSIAN_RADIXOR",
|
||||
"SNOWBALL_RUSSIAN_DIRECT",
|
||||
"SNOWBALL_RUSSIAN_LUCENE_FILTER",
|
||||
"SNOWBALL_SPANISH_RADIXOR",
|
||||
"SNOWBALL_SPANISH_DIRECT",
|
||||
"SNOWBALL_SPANISH_LUCENE_FILTER",
|
||||
"SNOWBALL_SWEDISH_RADIXOR",
|
||||
"SNOWBALL_SWEDISH_DIRECT",
|
||||
"SNOWBALL_SWEDISH_LUCENE_FILTER",
|
||||
"SNOWBALL_YIDDISH_RADIXOR",
|
||||
"SNOWBALL_YIDDISH_DIRECT",
|
||||
"SNOWBALL_YIDDISH_LUCENE_FILTER"
|
||||
})
|
||||
@@ -343,6 +330,7 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
GERMAN_LUCENE_GERMAN_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
|
||||
GERMAN_LUCENE_GERMAN_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
|
||||
GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.DE_DE),
|
||||
GERMAN_CISTEM(StemmerPatchTrieLoader.Language.DE_DE),
|
||||
SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES),
|
||||
SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES),
|
||||
SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER(StemmerPatchTrieLoader.Language.ES_ES),
|
||||
@@ -377,52 +365,36 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA),
|
||||
UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA),
|
||||
UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA),
|
||||
SNOWBALL_DANISH_RADIXOR(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
|
||||
SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
|
||||
SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
|
||||
SNOWBALL_DUTCH_RADIXOR(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
|
||||
SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
|
||||
SNOWBALL_DUTCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
|
||||
SNOWBALL_FINNISH_RADIXOR(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
|
||||
SNOWBALL_FINNISH_DIRECT(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
|
||||
SNOWBALL_FINNISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FI_FI, SnowballLanguageCase.FINNISH),
|
||||
SNOWBALL_FRENCH_RADIXOR(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
|
||||
SNOWBALL_FRENCH_DIRECT(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
|
||||
SNOWBALL_FRENCH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.FR_FR, SnowballLanguageCase.FRENCH),
|
||||
SNOWBALL_GERMAN_RADIXOR(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
|
||||
SNOWBALL_GERMAN_DIRECT(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
|
||||
SNOWBALL_GERMAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DE_DE, SnowballLanguageCase.GERMAN),
|
||||
SNOWBALL_HUNGARIAN_RADIXOR(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
|
||||
SNOWBALL_HUNGARIAN_DIRECT(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
|
||||
SNOWBALL_HUNGARIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.HU_HU, SnowballLanguageCase.HUNGARIAN),
|
||||
SNOWBALL_ITALIAN_RADIXOR(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
|
||||
SNOWBALL_ITALIAN_DIRECT(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
|
||||
SNOWBALL_ITALIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.IT_IT, SnowballLanguageCase.ITALIAN),
|
||||
SNOWBALL_NORWEGIAN_BOKMAL_RADIXOR(StemmerPatchTrieLoader.Language.NB_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_BOKMAL),
|
||||
SNOWBALL_NORWEGIAN_BOKMAL_DIRECT(StemmerPatchTrieLoader.Language.NB_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_BOKMAL),
|
||||
SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NB_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_BOKMAL),
|
||||
SNOWBALL_NORWEGIAN_NYNORSK_RADIXOR(StemmerPatchTrieLoader.Language.NN_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_NYNORSK),
|
||||
SNOWBALL_NORWEGIAN_NYNORSK_DIRECT(StemmerPatchTrieLoader.Language.NN_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_NYNORSK),
|
||||
SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO,
|
||||
SnowballLanguageCase.NORWEGIAN_NYNORSK),
|
||||
SNOWBALL_PORTUGUESE_RADIXOR(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
|
||||
SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
|
||||
SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
|
||||
SNOWBALL_RUSSIAN_RADIXOR(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
|
||||
SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
|
||||
SNOWBALL_RUSSIAN_LUCENE_FILTER(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),
|
||||
SNOWBALL_SPANISH_RADIXOR(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
|
||||
SNOWBALL_SPANISH_DIRECT(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
|
||||
SNOWBALL_SPANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.ES_ES, SnowballLanguageCase.SPANISH),
|
||||
SNOWBALL_SWEDISH_RADIXOR(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
|
||||
SNOWBALL_SWEDISH_DIRECT(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
|
||||
SNOWBALL_SWEDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.SV_SE, SnowballLanguageCase.SWEDISH),
|
||||
SNOWBALL_YIDDISH_RADIXOR(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH),
|
||||
SNOWBALL_YIDDISH_DIRECT(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH),
|
||||
SNOWBALL_YIDDISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.YI, SnowballLanguageCase.YIDDISH);
|
||||
|
||||
@@ -504,6 +476,7 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
tokenFilter(input -> new GermanLightStemFilter(germanNormalize(input)));
|
||||
case GERMAN_LUCENE_GERMAN_MINIMAL_STEM_FILTER ->
|
||||
tokenFilter(input -> new GermanMinimalStemFilter(germanNormalize(input)));
|
||||
case GERMAN_CISTEM -> direct(createGermanCistemStemmer());
|
||||
case SPANISH_LUCENE_SPANISH_LIGHT_STEM_FILTER ->
|
||||
tokenFilter(input -> new SpanishLightStemFilter(lowercase(input)));
|
||||
case SPANISH_LUCENE_SPANISH_MINIMAL_STEM_FILTER ->
|
||||
@@ -561,6 +534,15 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a CISTEM stemmer adapter.
|
||||
*
|
||||
* @return German stem function
|
||||
*/
|
||||
private static Stemmer createGermanCistemStemmer() {
|
||||
return Cistem::stem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct stemmer function.
|
||||
*/
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user