Refresh multilingual benchmarks and fix overlapping gold evaluation
Recompute published benchmark results for all default language models, exclude Polish Polimorf, add Hebrew documentation, and record the current benchmark environment. Evaluate repeated surface forms as an overlapping gold cover and publish only applicable metrics for candidate policies.
This commit is contained in:
@@ -0,0 +1,195 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TreeMap;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.egothor.stemmer.CaseProcessingMode;
|
||||
import org.egothor.stemmer.CompiledPatchCommand;
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.ReductionMode;
|
||||
import org.egothor.stemmer.StemmerDictionaryParser;
|
||||
import org.egothor.stemmer.StemmerModelDescriptor;
|
||||
import org.egothor.stemmer.StemmerModelRegistry;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
/**
|
||||
* Writes deterministic corpus and preferred patch-command counts for every
|
||||
* registered default model.
|
||||
*
|
||||
* <p>
|
||||
* This application performs setup-time analysis only; it does not publish or
|
||||
* interpret runtime performance. Optional model variants are excluded by
|
||||
* resolving every entry through
|
||||
* {@link StemmerModelRegistry#requireDefault(StemmerPatchTrieLoader.Language)}.
|
||||
* </p>
|
||||
*/
|
||||
public final class BenchmarkCorpusReportApplication {
|
||||
|
||||
/**
|
||||
* Utility class.
|
||||
*/
|
||||
private BenchmarkCorpusReportApplication() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one UTF-8 CSV report.
|
||||
*
|
||||
* @param arguments one output-file path
|
||||
* @throws IOException if model discovery, dictionary parsing, trie loading, or
|
||||
* report writing fails
|
||||
*/
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
if (arguments.length != 1) {
|
||||
throw new IllegalArgumentException("Expected one corpus-report output path.");
|
||||
}
|
||||
|
||||
final Path output = Path.of(arguments[0]);
|
||||
final Path parent = output.toAbsolutePath().getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
|
||||
final StringBuilder csv = new StringBuilder(16_384);
|
||||
csv.append("Language,Model ID,Model version,Model SHA-256,Dictionary rows,Total tokens,Already-root tokens,Changed tokens,")
|
||||
.append("Speed timing tokens,All exact matches,Changed exact matches,Root preserved matches,")
|
||||
.append("Command class,Command count\n");
|
||||
final StemmerModelRegistry registry = StemmerModelRegistry.fromContextClassLoader();
|
||||
for (StemmerPatchTrieLoader.Language language : StemmerPatchTrieLoader.Language.values()) {
|
||||
appendLanguage(csv, registry, language);
|
||||
}
|
||||
Files.writeString(output, csv, StandardCharsets.UTF_8);
|
||||
System.out.println("Benchmark corpus report: " + output.toAbsolutePath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends all command-class rows for one default model.
|
||||
*
|
||||
* @param csv destination
|
||||
* @param registry discovered model registry
|
||||
* @param language language to analyze
|
||||
* @throws IOException if the model cannot be parsed or loaded
|
||||
*/
|
||||
private static void appendLanguage(final StringBuilder csv, final StemmerModelRegistry registry,
|
||||
final StemmerPatchTrieLoader.Language language) throws IOException {
|
||||
final StemmerModelDescriptor descriptor = registry.requireDefault(language);
|
||||
final int dictionaryRows = countDictionaryRows(descriptor);
|
||||
final LanguageBenchmarkCorpus.Corpus corpus = LanguageBenchmarkCorpus.createFullCorpus(language);
|
||||
final String[] tokens = corpus.tokens();
|
||||
final String[] expectedRoots = corpus.expectedRoots();
|
||||
long alreadyRootTokens = 0;
|
||||
for (int index = 0; index < tokens.length; index++) {
|
||||
if (Objects.equals(tokens[index], expectedRoots[index])) {
|
||||
alreadyRootTokens++;
|
||||
}
|
||||
}
|
||||
final long changedTokens = tokens.length - alreadyRootTokens;
|
||||
final int timingTokens = LanguageBenchmarkCorpus.createChangedCorpus(language).tokens().length;
|
||||
|
||||
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(language, true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
final RadixorBenchmarkStemmer stemmer = new RadixorBenchmarkStemmer(trie);
|
||||
final Map<String, Long> commandCounts = new TreeMap<>();
|
||||
long allExactMatches = 0;
|
||||
long changedExactMatches = 0;
|
||||
long rootPreservedMatches = 0;
|
||||
for (int index = 0; index < tokens.length; index++) {
|
||||
final String token = tokens[index];
|
||||
final String expectedRoot = expectedRoots[index];
|
||||
final CompiledPatchCommand command = trie.getNormalizedString(token);
|
||||
final String commandClass = command == null ? "NoCommand" : command.getClass().getSimpleName();
|
||||
commandCounts.merge(commandClass, 1L, Math::addExact);
|
||||
final String actualRoot = stemmer.stem(token);
|
||||
if (Objects.equals(actualRoot, expectedRoot)) {
|
||||
allExactMatches++;
|
||||
if (Objects.equals(token, expectedRoot)) {
|
||||
rootPreservedMatches++;
|
||||
} else {
|
||||
changedExactMatches++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, Long> commandCount : commandCounts.entrySet()) {
|
||||
csv.append(language).append(',')
|
||||
.append(descriptor.id()).append(',')
|
||||
.append(descriptor.version()).append(',')
|
||||
.append(descriptor.sha256()).append(',')
|
||||
.append(dictionaryRows).append(',')
|
||||
.append(tokens.length).append(',')
|
||||
.append(alreadyRootTokens).append(',')
|
||||
.append(changedTokens).append(',')
|
||||
.append(timingTokens).append(',')
|
||||
.append(allExactMatches).append(',')
|
||||
.append(changedExactMatches).append(',')
|
||||
.append(rootPreservedMatches).append(',')
|
||||
.append(commandCount.getKey()).append(',')
|
||||
.append(commandCount.getValue()).append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts valid logical rows in one default dictionary.
|
||||
*
|
||||
* @param descriptor model descriptor
|
||||
* @return parsed dictionary-row count
|
||||
* @throws IOException if the dictionary cannot be opened or parsed
|
||||
*/
|
||||
private static int countDictionaryRows(final StemmerModelDescriptor descriptor) throws IOException {
|
||||
final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
|
||||
final ClassLoader classLoader = contextClassLoader == null
|
||||
? BenchmarkCorpusReportApplication.class.getClassLoader()
|
||||
: contextClassLoader;
|
||||
final InputStream resource = classLoader.getResourceAsStream(descriptor.resource());
|
||||
if (resource == null) {
|
||||
throw new IOException("Dictionary resource is missing for model " + descriptor.id() + ": "
|
||||
+ descriptor.resource() + ".");
|
||||
}
|
||||
final int[] rows = {0};
|
||||
try (InputStream raw = resource;
|
||||
GZIPInputStream gzip = new GZIPInputStream(raw);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
|
||||
StemmerDictionaryParser.parse(reader, descriptor.resource(), CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
(stem, variants, lineNumber) -> rows[0] = Math.addExact(rows[0], 1));
|
||||
}
|
||||
return rows[0];
|
||||
}
|
||||
}
|
||||
@@ -37,12 +37,15 @@ import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.egothor.stemmer.StemmerModelDescriptor;
|
||||
import org.egothor.stemmer.StemmerModelRegistry;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
/**
|
||||
@@ -75,6 +78,11 @@ final class LanguageBenchmarkCorpus {
|
||||
private static final Map<StemmerPatchTrieLoader.Language, Corpus> CHANGED_TIMING_CORPORA =
|
||||
new EnumMap<>(StemmerPatchTrieLoader.Language.class);
|
||||
|
||||
/**
|
||||
* Shared changed-token timing corpora keyed by explicit bundled model ID.
|
||||
*/
|
||||
private static final Map<String, Corpus> MODEL_CHANGED_TIMING_CORPORA = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Shared complete corpora keyed by bundled Radixor language.
|
||||
*/
|
||||
@@ -107,6 +115,25 @@ final class LanguageBenchmarkCorpus {
|
||||
return createChangedCorpus(language).tokens();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a deterministic changed-token timing corpus from an explicitly
|
||||
* selected bundled model dictionary.
|
||||
*
|
||||
* <p>
|
||||
* Only token/root pairs where the token differs from the expected root are
|
||||
* included. Smaller changed-token resources are repeated in stable order until
|
||||
* the timing corpus reaches 5,000 tokens.
|
||||
* </p>
|
||||
*
|
||||
* @param modelId exact bundled model identifier
|
||||
* @return token array containing changed-token dictionary entries, repeated
|
||||
* only when the changed-token resource is smaller than 5,000 tokens
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
static String[] createTokens(final String modelId) throws IOException {
|
||||
return createChangedCorpus(modelId).tokens();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a deterministic changed-token timing corpus from a bundled language
|
||||
* dictionary.
|
||||
@@ -119,6 +146,18 @@ final class LanguageBenchmarkCorpus {
|
||||
return cachedChangedCorpus(language);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a deterministic changed-token timing corpus from an explicitly
|
||||
* selected bundled model dictionary.
|
||||
*
|
||||
* @param modelId exact bundled model identifier
|
||||
* @return changed-token corpus with expected roots
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
static Corpus createChangedCorpus(final String modelId) throws IOException {
|
||||
return cachedChangedCorpus(modelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a deterministic full-dictionary timing corpus and expected root
|
||||
* array from a bundled language dictionary.
|
||||
@@ -220,6 +259,29 @@ final class LanguageBenchmarkCorpus {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a cached changed-token timing corpus, creating it once per JVM when
|
||||
* necessary.
|
||||
*
|
||||
* @param modelId exact bundled model identifier
|
||||
* @return changed-token timing corpus
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
private static Corpus cachedChangedCorpus(final String modelId) throws IOException {
|
||||
Objects.requireNonNull(modelId, "modelId");
|
||||
|
||||
synchronized (LanguageBenchmarkCorpus.class) {
|
||||
final Corpus existing = MODEL_CHANGED_TIMING_CORPORA.get(modelId);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
final Corpus created = buildChangedTimingCorpus(modelId, MINIMUM_TIMING_TOKEN_COUNT);
|
||||
MODEL_CHANGED_TIMING_CORPORA.put(modelId, created);
|
||||
return created;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a deterministic timing corpus from a bundled language dictionary.
|
||||
*
|
||||
@@ -263,11 +325,41 @@ final class LanguageBenchmarkCorpus {
|
||||
private static Corpus buildChangedTimingCorpus(final StemmerPatchTrieLoader.Language language,
|
||||
final int minimumTokenCount) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
return buildChangedTimingCorpus(readCandidates(language, Integer.MAX_VALUE), language.toString(),
|
||||
minimumTokenCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a deterministic changed-token timing corpus from an explicitly
|
||||
* selected bundled model dictionary.
|
||||
*
|
||||
* @param modelId exact bundled model identifier
|
||||
* @param minimumTokenCount minimum token count for timing
|
||||
* @return changed-token corpus with expected roots
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
private static Corpus buildChangedTimingCorpus(final String modelId, final int minimumTokenCount)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(modelId, "modelId");
|
||||
return buildChangedTimingCorpus(readCandidates(modelId, Integer.MAX_VALUE), modelId, minimumTokenCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a deterministic changed-token timing corpus from parsed entries.
|
||||
*
|
||||
* @param allCandidates all valid dictionary entries
|
||||
* @param sourceLabel human-readable source label for diagnostics
|
||||
* @param minimumTokenCount minimum token count for timing
|
||||
* @return changed-token corpus with expected roots
|
||||
*/
|
||||
private static Corpus buildChangedTimingCorpus(final List<Entry> allCandidates, final String sourceLabel,
|
||||
final int minimumTokenCount) {
|
||||
Objects.requireNonNull(allCandidates, "allCandidates");
|
||||
Objects.requireNonNull(sourceLabel, "sourceLabel");
|
||||
if (minimumTokenCount < 1) {
|
||||
throw new IllegalArgumentException("minimumTokenCount must be at least 1.");
|
||||
}
|
||||
|
||||
final List<Entry> allCandidates = readCandidates(language, Integer.MAX_VALUE);
|
||||
final List<Entry> changedCandidates = new ArrayList<>(allCandidates.size());
|
||||
for (Entry entry : allCandidates) {
|
||||
if (!Objects.equals(entry.token(), entry.root())) {
|
||||
@@ -276,7 +368,7 @@ final class LanguageBenchmarkCorpus {
|
||||
}
|
||||
if (changedCandidates.isEmpty()) {
|
||||
throw new IllegalStateException("No changed-token benchmark corpus tokens were available for "
|
||||
+ language + ".");
|
||||
+ sourceLabel + ".");
|
||||
}
|
||||
|
||||
final int timingTokenCount = Math.max(changedCandidates.size(), minimumTokenCount);
|
||||
@@ -332,8 +424,35 @@ final class LanguageBenchmarkCorpus {
|
||||
*/
|
||||
private static List<Entry> readCandidates(final StemmerPatchTrieLoader.Language language, final int maximumTokenCount)
|
||||
throws IOException {
|
||||
final String resourcePath = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
|
||||
.requireDefault(language).resource();
|
||||
final StemmerModelDescriptor descriptor = StemmerModelRegistry.fromContextClassLoader()
|
||||
.requireDefault(language);
|
||||
return readCandidatesFromResource(descriptor.resource(), maximumTokenCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads token candidates from an explicitly selected bundled compressed
|
||||
* dictionary.
|
||||
*
|
||||
* @param modelId exact bundled model identifier
|
||||
* @param maximumTokenCount maximum token count to read
|
||||
* @return deterministic candidate list
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
private static List<Entry> readCandidates(final String modelId, final int maximumTokenCount) throws IOException {
|
||||
final StemmerModelDescriptor descriptor = StemmerModelRegistry.fromContextClassLoader().require(modelId);
|
||||
return readCandidatesFromResource(descriptor.resource(), maximumTokenCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads token candidates from a bundled compressed dictionary resource.
|
||||
*
|
||||
* @param resourcePath classpath resource path
|
||||
* @param maximumTokenCount maximum token count to read
|
||||
* @return deterministic candidate list
|
||||
* @throws IOException if the resource cannot be read
|
||||
*/
|
||||
private static List<Entry> readCandidatesFromResource(final String resourcePath, final int maximumTokenCount)
|
||||
throws IOException {
|
||||
final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath);
|
||||
if (resource == null) {
|
||||
throw new IllegalStateException("Missing bundled benchmark resource " + resourcePath + ".");
|
||||
|
||||
@@ -143,6 +143,11 @@ public class MultiLanguageStemmerComparisonBenchmark {
|
||||
*/
|
||||
private LanguageState french;
|
||||
|
||||
/**
|
||||
* Hebrew benchmark state.
|
||||
*/
|
||||
private LanguageState hebrew;
|
||||
|
||||
/**
|
||||
* Hungarian benchmark state.
|
||||
*/
|
||||
@@ -196,6 +201,7 @@ public class MultiLanguageStemmerComparisonBenchmark {
|
||||
this.persian = load(StemmerPatchTrieLoader.Language.FA_IR);
|
||||
this.finnish = load(StemmerPatchTrieLoader.Language.FI_FI);
|
||||
this.french = load(StemmerPatchTrieLoader.Language.FR_FR);
|
||||
this.hebrew = load(StemmerPatchTrieLoader.Language.HE_IL);
|
||||
this.hungarian = load(StemmerPatchTrieLoader.Language.HU_HU);
|
||||
this.italian = load(StemmerPatchTrieLoader.Language.IT_IT);
|
||||
this.norwegianBokmal = load(StemmerPatchTrieLoader.Language.NB_NO);
|
||||
@@ -600,6 +606,17 @@ public class MultiLanguageStemmerComparisonBenchmark {
|
||||
runRadixor(sharedState.french, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs Radixor over the Hebrew corpus.
|
||||
*
|
||||
* @param sharedState shared benchmark state
|
||||
* @param blackhole result sink
|
||||
*/
|
||||
@Benchmark
|
||||
public void hebrewRadixor(final SharedState sharedState, final Blackhole blackhole) {
|
||||
runRadixor(sharedState.hebrew, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs Lucene FrenchLightStemFilter over the French corpus.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.morfologik.MorfologikFilter;
|
||||
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
||||
import org.egothor.stemmer.CompiledPatchCommand;
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.ReductionMode;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
import org.openjdk.jmh.annotations.Benchmark;
|
||||
import org.openjdk.jmh.annotations.BenchmarkMode;
|
||||
import org.openjdk.jmh.annotations.Fork;
|
||||
import org.openjdk.jmh.annotations.Level;
|
||||
import org.openjdk.jmh.annotations.Measurement;
|
||||
import org.openjdk.jmh.annotations.Mode;
|
||||
import org.openjdk.jmh.annotations.OutputTimeUnit;
|
||||
import org.openjdk.jmh.annotations.Scope;
|
||||
import org.openjdk.jmh.annotations.Setup;
|
||||
import org.openjdk.jmh.annotations.State;
|
||||
import org.openjdk.jmh.annotations.Warmup;
|
||||
import org.openjdk.jmh.infra.Blackhole;
|
||||
|
||||
/**
|
||||
* Compares the two Polish production stemmer paths over the PoliMorf-backed
|
||||
* Radixor dictionary workload.
|
||||
*
|
||||
* <p>
|
||||
* Each benchmark operation processes the same changed-token corpus derived from
|
||||
* {@code pl-pl-polimorf}. The Radixor method uses the explicit
|
||||
* {@code pl-pl-polimorf} runtime model, while the Lucene method uses the public
|
||||
* {@link MorfologikFilter} path.
|
||||
* </p>
|
||||
*/
|
||||
@BenchmarkMode(Mode.AverageTime)
|
||||
@OutputTimeUnit(TimeUnit.NANOSECONDS)
|
||||
@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS)
|
||||
@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
|
||||
@Fork(value = 1, jvmArgsAppend = { "-Xmx6g" })
|
||||
public class PolishPolimorfStemmerComparisonBenchmark {
|
||||
|
||||
/**
|
||||
* Explicit Polish PoliMorf model identifier.
|
||||
*/
|
||||
private static final String POLIMORF_MODEL_ID = "pl-pl-polimorf";
|
||||
|
||||
/**
|
||||
* Shared PoliMorf corpus and Radixor trie state.
|
||||
*/
|
||||
@State(Scope.Benchmark)
|
||||
public static class SharedState {
|
||||
|
||||
/**
|
||||
* Shared deterministic changed-token dictionary corpus.
|
||||
*/
|
||||
private String[] tokens;
|
||||
|
||||
/**
|
||||
* Radixor benchmark adapter over the PoliMorf model.
|
||||
*/
|
||||
private RadixorBenchmarkStemmer radixorStemmer;
|
||||
|
||||
/**
|
||||
* Initializes the PoliMorf corpus and Radixor trie before measurement.
|
||||
*
|
||||
* @throws IOException if the corpus or trie cannot be loaded
|
||||
*/
|
||||
@Setup(Level.Trial)
|
||||
public void setUp() throws IOException {
|
||||
this.tokens = LanguageBenchmarkCorpus.createTokens(POLIMORF_MODEL_ID);
|
||||
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(POLIMORF_MODEL_ID,
|
||||
true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
this.radixorStemmer = new RadixorBenchmarkStemmer(trie);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-thread Lucene Morfologik filter state.
|
||||
*/
|
||||
@State(Scope.Thread)
|
||||
public static class LuceneFilterState {
|
||||
|
||||
/**
|
||||
* Reusable Morfologik token-filter pipeline.
|
||||
*/
|
||||
private final FilterPipeline polishMorfologik = new FilterPipeline(MorfologikFilter::new);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs Radixor with the {@code pl-pl-polimorf} model over the PoliMorf corpus.
|
||||
*
|
||||
* @param sharedState shared benchmark state
|
||||
* @param blackhole result sink
|
||||
*/
|
||||
@Benchmark
|
||||
public void polishPolimorfRadixor(final SharedState sharedState, final Blackhole blackhole) {
|
||||
final String[] tokens = sharedState.tokens;
|
||||
final RadixorBenchmarkStemmer stemmer = sharedState.radixorStemmer;
|
||||
|
||||
for (String token : tokens) {
|
||||
blackhole.consume(stemmer.stem(token));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs Lucene MorfologikFilter over the PoliMorf-derived corpus.
|
||||
*
|
||||
* @param sharedState shared benchmark state
|
||||
* @param filterState reusable filter state
|
||||
* @param blackhole result sink
|
||||
* @throws IOException if Lucene token streaming fails
|
||||
*/
|
||||
@Benchmark
|
||||
public void polishLuceneMorfologikFilter(final SharedState sharedState, final LuceneFilterState filterState,
|
||||
final Blackhole blackhole) throws IOException {
|
||||
filterState.polishMorfologik.run(sharedState.tokens, blackhole);
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory for a Lucene filter under test.
|
||||
*/
|
||||
private interface FilterFactory {
|
||||
|
||||
/**
|
||||
* Creates a token stream wrapping the supplied benchmark input stream.
|
||||
*
|
||||
* @param input input token stream
|
||||
* @return filter stream
|
||||
*/
|
||||
TokenStream create(TokenStream input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reusable input stream, filter stream, and term attribute for one Lucene
|
||||
* benchmark method.
|
||||
*/
|
||||
private static final class FilterPipeline {
|
||||
|
||||
/**
|
||||
* Reusable benchmark input stream.
|
||||
*/
|
||||
private final BenchmarkTokenStream input;
|
||||
|
||||
/**
|
||||
* Lucene filter output stream.
|
||||
*/
|
||||
private final TokenStream output;
|
||||
|
||||
/**
|
||||
* Term attribute consumed by the benchmark.
|
||||
*/
|
||||
private final CharTermAttribute termAttribute;
|
||||
|
||||
/**
|
||||
* Creates one reusable filter pipeline.
|
||||
*
|
||||
* @param factory filter factory
|
||||
*/
|
||||
private FilterPipeline(final FilterFactory factory) {
|
||||
this.input = new BenchmarkTokenStream(new String[0]);
|
||||
this.output = factory.create(this.input);
|
||||
this.termAttribute = this.output.addAttribute(CharTermAttribute.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the filter over one token corpus and consumes all emitted terms.
|
||||
*
|
||||
* @param tokens token corpus
|
||||
* @param blackhole result sink
|
||||
* @throws IOException if Lucene token streaming fails
|
||||
*/
|
||||
private void run(final String[] tokens, final Blackhole blackhole) throws IOException {
|
||||
this.input.setTokens(tokens);
|
||||
this.output.reset();
|
||||
while (this.output.incrementToken()) {
|
||||
blackhole.consume(this.termAttribute.toString());
|
||||
}
|
||||
this.output.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,9 @@ import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
|
||||
/** Authoritative analytical view of the candidate matrix defined by the JMH quality benchmark. */
|
||||
public final class QualityStemmerMatrix {
|
||||
/** Optional Polish PoliMorf model included as an explicit non-default comparison. */
|
||||
private static final String POLISH_POLIMORF_MODEL_ID = "pl-pl-polimorf";
|
||||
|
||||
/** Utility class. */
|
||||
private QualityStemmerMatrix() {
|
||||
throw new AssertionError("No instances.");
|
||||
@@ -81,6 +84,15 @@ public final class QualityStemmerMatrix {
|
||||
@Override public boolean supportsMultipleOutputs() { return true; }
|
||||
}))
|
||||
.forEach(candidates::add);
|
||||
final List<Candidate> defaultPolishCandidates = candidates.stream()
|
||||
.filter(candidate -> candidate.language() == Language.PL_PL)
|
||||
.toList();
|
||||
candidates.add(new Candidate("POLISH_POLIMORF_RADIXOR", Language.PL_PL,
|
||||
POLISH_POLIMORF_MODEL_ID, POLISH_POLIMORF_MODEL_ID,
|
||||
() -> adapt(StemmerComparisonBenchmarkQuality.createRadixorQualityStemmer(POLISH_POLIMORF_MODEL_ID))));
|
||||
defaultPolishCandidates
|
||||
.forEach(candidate -> candidates.add(new Candidate(candidate.name(), candidate.language(),
|
||||
POLISH_POLIMORF_MODEL_ID, POLISH_POLIMORF_MODEL_ID, candidate.factory)));
|
||||
return List.copyOf(candidates);
|
||||
}
|
||||
|
||||
@@ -102,13 +114,23 @@ public final class QualityStemmerMatrix {
|
||||
public static final class Candidate {
|
||||
private final String name;
|
||||
private final Language language;
|
||||
private final String resultLanguage;
|
||||
private final String dictionaryModelId;
|
||||
private final StemmerFactory factory;
|
||||
|
||||
/** Creates an immutable facade over one benchmark candidate. */
|
||||
private Candidate(final String name, final Language language,
|
||||
final StemmerFactory factory) {
|
||||
this(name, language, language.name(), language.defaultModelId(), factory);
|
||||
}
|
||||
|
||||
/** Creates an immutable facade over one benchmark candidate and dictionary model. */
|
||||
private Candidate(final String name, final Language language, final String resultLanguage,
|
||||
final String dictionaryModelId, final StemmerFactory factory) {
|
||||
this.name = Objects.requireNonNull(name, "name");
|
||||
this.language = Objects.requireNonNull(language, "language");
|
||||
this.resultLanguage = Objects.requireNonNull(resultLanguage, "resultLanguage");
|
||||
this.dictionaryModelId = Objects.requireNonNull(dictionaryModelId, "dictionaryModelId");
|
||||
this.factory = Objects.requireNonNull(factory, "factory");
|
||||
}
|
||||
|
||||
@@ -122,6 +144,16 @@ public final class QualityStemmerMatrix {
|
||||
return this.language;
|
||||
}
|
||||
|
||||
/** @return stable report language or model label */
|
||||
public String resultLanguage() {
|
||||
return this.resultLanguage;
|
||||
}
|
||||
|
||||
/** @return exact dictionary model used as the gold-standard grouping source */
|
||||
public String dictionaryModelId() {
|
||||
return this.dictionaryModelId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a scenario-confined adapter using exactly the JMH factory and preprocessing path.
|
||||
*
|
||||
|
||||
@@ -643,6 +643,19 @@ public class StemmerComparisonBenchmarkQuality {
|
||||
return radixor(createRadixorStemmer(language));
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the authoritative multi-output Radixor adapter for an explicitly
|
||||
* selected runtime model.
|
||||
*
|
||||
* @param modelId exact model identifier
|
||||
* @return scenario-confined adapter using the JMH invocation path
|
||||
* @throws IOException if the compiled dictionary cannot be loaded
|
||||
*/
|
||||
static CandidateStemmer createRadixorQualityStemmer(final String modelId) throws IOException {
|
||||
return radixor(new RadixorBenchmarkStemmer(StemmerPatchTrieLoader.loadCompiled(
|
||||
modelId, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact-root agreement counters for one quality operation.
|
||||
*
|
||||
|
||||
@@ -42,6 +42,8 @@ import java.util.Objects;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.egothor.stemmer.CaseProcessingMode;
|
||||
import org.egothor.stemmer.StemmerModelDescriptor;
|
||||
import org.egothor.stemmer.StemmerModelRegistry;
|
||||
import org.egothor.stemmer.StemmerDictionaryParser;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
|
||||
@@ -58,10 +60,21 @@ public final class BundledGoldStandardLoader {
|
||||
*/
|
||||
public static List<GoldStandardGroup> load(final Language language) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
final String resource = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
|
||||
.requireDefault(language).resource();
|
||||
return loadModel(StemmerModelRegistry.fromContextClassLoader().requireDefault(language).id());
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses one explicitly selected compressed UTF-8 model dictionary with case preserved.
|
||||
* @param modelId exact model identifier
|
||||
* @return immutable groups in source-row order
|
||||
* @throws IOException if the resource is absent, malformed, or unreadable
|
||||
*/
|
||||
public static List<GoldStandardGroup> loadModel(final String modelId) throws IOException {
|
||||
Objects.requireNonNull(modelId, "modelId");
|
||||
final StemmerModelDescriptor descriptor = StemmerModelRegistry.fromContextClassLoader().require(modelId);
|
||||
final String resource = descriptor.resource();
|
||||
final List<GoldStandardGroup> groups = new ArrayList<>();
|
||||
try (InputStream raw = openResource(language, resource); InputStream gzip = new GZIPInputStream(raw);
|
||||
try (InputStream raw = openResource(modelId, resource); InputStream gzip = new GZIPInputStream(raw);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
|
||||
StemmerDictionaryParser.parse(reader, resource, CaseProcessingMode.AS_IS, (stem, variants, row) -> {
|
||||
final List<String> forms = new ArrayList<>(variants.length + 1);
|
||||
@@ -70,7 +83,7 @@ public final class BundledGoldStandardLoader {
|
||||
try {
|
||||
groups.add(new GoldStandardGroup(row, forms));
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IOException("Invalid dictionary group for language " + language + ", resource "
|
||||
throw new IOException("Invalid dictionary group for model " + modelId + ", resource "
|
||||
+ resource + ", row " + row + ": " + exception.getMessage(), exception);
|
||||
}
|
||||
});
|
||||
@@ -79,10 +92,10 @@ public final class BundledGoldStandardLoader {
|
||||
}
|
||||
|
||||
/** Opens one required classpath resource with a precise language diagnostic. */
|
||||
private static InputStream openResource(final Language language, final String resource) throws IOException {
|
||||
private static InputStream openResource(final String modelId, final String resource) throws IOException {
|
||||
final InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource);
|
||||
if (input == null) {
|
||||
throw new IOException("Dictionary resource is missing for language " + language + ": " + resource + ".");
|
||||
throw new IOException("Dictionary resource is missing for model " + modelId + ": " + resource + ".");
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
@@ -57,20 +57,8 @@ final class CandidateAwareEvaluator {
|
||||
if (policy == OutputPolicy.PRIMARY_OUTPUT) {
|
||||
throw new IllegalArgumentException("Candidate-aware evaluation requires ANY_CANDIDATE or ALL_CANDIDATES.");
|
||||
}
|
||||
final List<GoldStandardGroup> includedGroups = groups.stream().filter(group -> mode.includes(group.forms())).toList();
|
||||
final List<String> forms = new ArrayList<>();
|
||||
final List<Integer> groupIndexes = new ArrayList<>();
|
||||
long singletonRows = 0;
|
||||
long pairRows = 0;
|
||||
long underPossible = 0;
|
||||
for (int groupIndex = 0; groupIndex < includedGroups.size(); groupIndex++) {
|
||||
final GoldStandardGroup group = includedGroups.get(groupIndex);
|
||||
if (group.forms().size() == 1) { singletonRows = add(singletonRows, 1, "singleton rows"); }
|
||||
else { pairRows = add(pairRows, 1, "rows contributing under-stemming pairs"); }
|
||||
underPossible = add(underPossible, QualityEvaluator.chooseTwo(group.forms().size()), "under denominator");
|
||||
for (String form : group.forms()) { forms.add(form); groupIndexes.add(groupIndex); }
|
||||
}
|
||||
final String[] input = forms.toArray(String[]::new);
|
||||
final GoldStandardCover cover = GoldStandardCover.create(groups, mode);
|
||||
final String[] input = cover.forms().toArray(String[]::new);
|
||||
final String[] primary = stemmer.stem(input);
|
||||
final List<List<String>> rawCandidates = stemmer.stemCandidates(input);
|
||||
if (primary == null || primary.length != input.length || rawCandidates == null || rawCandidates.size() != input.length) {
|
||||
@@ -83,22 +71,31 @@ final class CandidateAwareEvaluator {
|
||||
long multipleCandidates = 0;
|
||||
long maximumCandidates = 0;
|
||||
long assignments = 0;
|
||||
final Signature[] formSignatures = new Signature[input.length];
|
||||
for (int index = 0; index < input.length; index++) {
|
||||
final Signature signature = signature(rawCandidates.get(index), primary[index], stemmerName, language,
|
||||
mode, policy, includedGroups.get(groupIndexes.get(index)).rowNumber(), input[index]);
|
||||
mode, policy, cover.representativeRow(index), input[index]);
|
||||
formSignatures[index] = signature;
|
||||
final int size = signature.candidates().size();
|
||||
if (size == 1) { oneCandidate = add(oneCandidate, 1, "single-candidate forms"); }
|
||||
else { multipleCandidates = add(multipleCandidates, 1, "multi-candidate forms"); }
|
||||
maximumCandidates = Math.max(maximumCandidates, size);
|
||||
assignments = add(assignments, size, "candidate assignments");
|
||||
distinctCandidates.addAll(signature.candidates());
|
||||
counts.computeIfAbsent(signature, ignored -> new SignatureCount()).increment(groupIndexes.get(index));
|
||||
counts.computeIfAbsent(signature, ignored -> new SignatureCount()).incrementTotal();
|
||||
}
|
||||
for (int groupIndex = 0; groupIndex < cover.groups().size(); groupIndex++) {
|
||||
for (String form : cover.groups().get(groupIndex).forms()) {
|
||||
counts.get(formSignatures[cover.indexOf(form)]).incrementGroup(groupIndex);
|
||||
}
|
||||
}
|
||||
|
||||
final List<Map.Entry<Signature, SignatureCount>> signatures = new ArrayList<>(counts.entrySet());
|
||||
signatures.sort(Map.Entry.comparingByKey());
|
||||
long sameGroupRelated = 0;
|
||||
long crossGroupRelated = 0;
|
||||
long globallyRelated = 0;
|
||||
long forcedSameGroupRelated = 0;
|
||||
long globallyForcedRelated = 0;
|
||||
final Map<String, List<Integer>> inverted = new HashMap<>();
|
||||
for (int index = 0; index < signatures.size(); index++) {
|
||||
final Map.Entry<Signature, SignatureCount> entry = signatures.get(index);
|
||||
@@ -107,10 +104,13 @@ final class CandidateAwareEvaluator {
|
||||
sameWithin = add(sameWithin, QualityEvaluator.chooseTwo(groupCount), "same-signature group pairs");
|
||||
}
|
||||
sameGroupRelated = add(sameGroupRelated, sameWithin, "same-group related pairs");
|
||||
if (policy == OutputPolicy.ALL_CANDIDATES || entry.getKey().candidates().size() == 1) {
|
||||
crossGroupRelated = add(crossGroupRelated,
|
||||
subtract(QualityEvaluator.chooseTwo(entry.getValue().total()), sameWithin, "same-signature cross pairs"),
|
||||
"cross-group related pairs");
|
||||
globallyRelated = add(globallyRelated, QualityEvaluator.chooseTwo(entry.getValue().total()),
|
||||
"globally related pairs");
|
||||
if (entry.getKey().candidates().size() == 1) {
|
||||
forcedSameGroupRelated = add(forcedSameGroupRelated, sameWithin,
|
||||
"forced same-group related pairs");
|
||||
globallyForcedRelated = add(globallyForcedRelated,
|
||||
QualityEvaluator.chooseTwo(entry.getValue().total()), "globally forced related pairs");
|
||||
}
|
||||
for (String candidate : entry.getKey().candidates()) {
|
||||
inverted.computeIfAbsent(candidate, ignored -> new ArrayList<>()).add(index);
|
||||
@@ -134,20 +134,58 @@ final class CandidateAwareEvaluator {
|
||||
}
|
||||
final long total = multiply(left.total(), right.total(), "different-signature pairs");
|
||||
sameGroupRelated = add(sameGroupRelated, same, "same-group related pairs");
|
||||
if (policy == OutputPolicy.ALL_CANDIDATES) {
|
||||
crossGroupRelated = add(crossGroupRelated, subtract(total, same, "different-signature cross pairs"),
|
||||
"cross-group related pairs");
|
||||
globallyRelated = add(globallyRelated, total, "globally related pairs");
|
||||
}
|
||||
for (GoldStandardCover.DuplicateRelation duplicate : cover.duplicateRelations()) {
|
||||
final Signature left = formSignatures[duplicate.leftFormIndex()];
|
||||
final Signature right = formSignatures[duplicate.rightFormIndex()];
|
||||
if (intersects(left, right)) {
|
||||
sameGroupRelated = subtract(sameGroupRelated, duplicate.extraOccurrences(),
|
||||
"duplicate same-group candidate relations");
|
||||
}
|
||||
if (isForcedCollision(left, right)) {
|
||||
forcedSameGroupRelated = subtract(forcedSameGroupRelated, duplicate.extraOccurrences(),
|
||||
"duplicate forced same-group candidate relations");
|
||||
}
|
||||
}
|
||||
final long wordCount = input.length;
|
||||
final long underPossible = cover.relatedPairs();
|
||||
final long overPossible = subtract(QualityEvaluator.chooseTwo(wordCount), underPossible, "over denominator");
|
||||
final long underError = subtract(underPossible, sameGroupRelated, "candidate under errors");
|
||||
final long overError = policy == OutputPolicy.ALL_CANDIDATES
|
||||
? subtract(globallyRelated, sameGroupRelated, "all-candidate over errors")
|
||||
: subtract(globallyForcedRelated, forcedSameGroupRelated, "any-candidate over errors");
|
||||
return new QualityResult(stemmerName, language, mode, policy,
|
||||
includedGroups.size(), wordCount, singletonRows, pairRows, oneCandidate, multipleCandidates,
|
||||
maximumCandidates, assignments, distinctCandidates.size(), crossGroupRelated, overPossible,
|
||||
cover.groups().size(), wordCount, cover.singletonRows(), cover.pairRows(),
|
||||
oneCandidate, multipleCandidates, maximumCandidates, assignments,
|
||||
distinctCandidates.size(), overError, overPossible,
|
||||
underError, underPossible, null);
|
||||
}
|
||||
|
||||
/** Returns whether two canonical candidate sets intersect. */
|
||||
private static boolean intersects(final Signature left, final Signature right) {
|
||||
int leftIndex = 0;
|
||||
int rightIndex = 0;
|
||||
while (leftIndex < left.candidates().size() && rightIndex < right.candidates().size()) {
|
||||
final int comparison = left.candidates().get(leftIndex).compareTo(right.candidates().get(rightIndex));
|
||||
if (comparison == 0) {
|
||||
return true;
|
||||
}
|
||||
if (comparison < 0) {
|
||||
leftIndex++;
|
||||
} else {
|
||||
rightIndex++;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Returns whether every independent selection forces the same output. */
|
||||
private static boolean isForcedCollision(final Signature left, final Signature right) {
|
||||
return left.candidates().size() == 1 && right.candidates().size() == 1
|
||||
&& left.candidates().getFirst().equals(right.candidates().getFirst());
|
||||
}
|
||||
|
||||
/** Canonicalizes and validates one adapter candidate collection. */
|
||||
private static Signature signature(final List<String> raw, final String primary, final String stemmer,
|
||||
final String language, final ProcessingMode mode, final OutputPolicy policy,
|
||||
@@ -182,11 +220,12 @@ final class CandidateAwareEvaluator {
|
||||
return Integer.compare(candidates.size(), other.candidates.size());
|
||||
}
|
||||
}
|
||||
/** Aggregated global and per-group frequency of one signature. */
|
||||
/** Aggregated unique-form and per-group membership frequency of one signature. */
|
||||
private static final class SignatureCount {
|
||||
private long total;
|
||||
private final Map<Integer, Long> byGroup = new HashMap<>();
|
||||
/** Adds one word occurrence. */ private void increment(final int group) { total = add(total, 1, "signature frequency"); byGroup.merge(group, 1L, (left, right) -> add(left, right, "signature group frequency")); }
|
||||
/** Adds one unique word form. */ private void incrementTotal() { total = add(total, 1, "signature frequency"); }
|
||||
/** Adds membership in one gold group. */ private void incrementGroup(final int group) { byGroup.merge(group, 1L, (left, right) -> add(left, right, "signature group frequency")); }
|
||||
/** @return global signature frequency */ private long total() { return total; }
|
||||
/** @return mutable internally owned per-group frequencies */ private Map<Integer, Long> byGroup() { return byGroup; }
|
||||
}
|
||||
|
||||
@@ -109,6 +109,24 @@ final class CandidateAwareEvaluatorTest {
|
||||
assertEquals(2, any.underErrorPairs()); assertEquals(any.underErrorPairs(), all.underErrorPairs());
|
||||
}
|
||||
|
||||
/** Verifies candidate relations over a gold cover with shared forms. */
|
||||
@Test @DisplayName("Candidate evaluation deduplicates forms and overlapping gold relations")
|
||||
void overlappingGoldCover() throws IOException {
|
||||
final List<GoldStandardGroup> groups = List.of(
|
||||
new GoldStandardGroup(1, List.of("a", "b")),
|
||||
new GoldStandardGroup(2, List.of("a", "b", "c")));
|
||||
final Map<String, String> primary = Map.of("a", "x", "b", "y", "c", "z");
|
||||
final Map<String, List<String>> candidates = Map.of(
|
||||
"a", List.of("x", "shared"), "b", List.of("y", "shared"), "c", List.of("z"));
|
||||
final QualityResult all = CandidateAwareEvaluator.evaluate("Synthetic", "MULTI",
|
||||
ProcessingMode.ALL_WORDS, OutputPolicy.ALL_CANDIDATES, groups, adapter(primary, candidates));
|
||||
assertEquals(3, all.processedWordForms());
|
||||
assertEquals(3, all.underPossiblePairs());
|
||||
assertEquals(2, all.underErrorPairs());
|
||||
assertEquals(0, all.overPossiblePairs());
|
||||
assertEquals(0, all.overErrorPairs());
|
||||
}
|
||||
|
||||
/** Compares the optimized signature algorithm with an independent fixed-seed oracle. */
|
||||
@Test @DisplayName("Optimized candidate metrics equal a deterministic randomized brute-force oracle")
|
||||
void randomizedOracleAgreement() throws IOException {
|
||||
@@ -118,12 +136,21 @@ final class CandidateAwareEvaluatorTest {
|
||||
final List<GoldStandardGroup> groups = new ArrayList<>();
|
||||
final Map<String, String> primary = new HashMap<>();
|
||||
final Map<String, List<String>> candidates = new HashMap<>();
|
||||
final List<String> existingForms = new ArrayList<>();
|
||||
int word = 0;
|
||||
for (int group = 0; group < groupCount; group++) {
|
||||
final List<String> forms = new ArrayList<>();
|
||||
for (int member = 0; member < 1 + random.nextInt(5); member++) {
|
||||
final String form = "w" + word++;
|
||||
final boolean reuse = !existingForms.isEmpty() && random.nextInt(5) == 0;
|
||||
final String form = reuse ? existingForms.get(random.nextInt(existingForms.size())) : "w" + word++;
|
||||
if (forms.contains(form)) {
|
||||
continue;
|
||||
}
|
||||
forms.add(form);
|
||||
if (reuse) {
|
||||
continue;
|
||||
}
|
||||
existingForms.add(form);
|
||||
final String primaryStem = "s" + random.nextInt(7);
|
||||
primary.put(form, primaryStem);
|
||||
final List<String> raw = new ArrayList<>();
|
||||
@@ -204,17 +231,21 @@ final class CandidateAwareEvaluatorTest {
|
||||
/** Enumerates small word pairs independently and returns under error/possible and over error/possible counts. */
|
||||
private static long[] oracle(final List<GoldStandardGroup> groups,
|
||||
final Map<String, List<String>> candidates) {
|
||||
final List<String> forms = new ArrayList<>();
|
||||
final List<Integer> labels = new ArrayList<>();
|
||||
final Map<String, Set<Integer>> memberships = new java.util.LinkedHashMap<>();
|
||||
for (int group = 0; group < groups.size(); group++) {
|
||||
for (String form : groups.get(group).forms()) { forms.add(form); labels.add(group); }
|
||||
for (String form : groups.get(group).forms()) {
|
||||
memberships.computeIfAbsent(form, ignored -> new LinkedHashSet<>()).add(group);
|
||||
}
|
||||
}
|
||||
final List<String> forms = List.copyOf(memberships.keySet());
|
||||
long underError = 0; long underPossible = 0; long overError = 0; long overPossible = 0; long anyOverError = 0;
|
||||
for (int left = 0; left < forms.size(); left++) {
|
||||
for (int right = left + 1; right < forms.size(); right++) {
|
||||
final Set<String> intersection = new LinkedHashSet<>(candidates.get(forms.get(left)));
|
||||
intersection.retainAll(new LinkedHashSet<>(candidates.get(forms.get(right))));
|
||||
if (labels.get(left).equals(labels.get(right))) {
|
||||
final Set<Integer> sharedGroups = new LinkedHashSet<>(memberships.get(forms.get(left)));
|
||||
sharedGroups.retainAll(memberships.get(forms.get(right)));
|
||||
if (!sharedGroups.isEmpty()) {
|
||||
underPossible++; if (intersection.isEmpty()) { underError++; }
|
||||
} else {
|
||||
overPossible++; if (!intersection.isEmpty()) { overError++; }
|
||||
|
||||
@@ -56,13 +56,18 @@ final class CandidateQualityAudit {
|
||||
static Scenario evaluate(final Candidate candidate, final ProcessingMode mode,
|
||||
final List<GoldStandardGroup> groups, final QualityResult primary, final QualityResult any,
|
||||
final int limit) throws IOException {
|
||||
final List<String> forms = new ArrayList<>();
|
||||
final List<Integer> groupIndexes = new ArrayList<>();
|
||||
final List<Integer> rows = new ArrayList<>();
|
||||
for (int group = 0; group < groups.size(); group++) {
|
||||
final GoldStandardGroup item = groups.get(group);
|
||||
if (!mode.includes(item.forms())) { continue; }
|
||||
for (String form : item.forms()) { forms.add(form); groupIndexes.add(group); rows.add(item.rowNumber()); }
|
||||
final GoldStandardCover cover = GoldStandardCover.create(groups, mode);
|
||||
final List<String> forms = cover.forms();
|
||||
final List<Integer> rows = new ArrayList<>(forms.size());
|
||||
final List<Set<Integer>> memberships = new ArrayList<>(forms.size());
|
||||
for (int index = 0; index < forms.size(); index++) {
|
||||
rows.add(cover.representativeRow(index));
|
||||
memberships.add(new HashSet<>());
|
||||
}
|
||||
for (int group = 0; group < cover.groups().size(); group++) {
|
||||
for (String form : cover.groups().get(group).forms()) {
|
||||
memberships.get(cover.indexOf(form)).add(group);
|
||||
}
|
||||
}
|
||||
final BatchStemmer stemmer = candidate.createStemmer();
|
||||
final String[] primaryOutputs = stemmer.stem(forms.toArray(String[]::new));
|
||||
@@ -78,7 +83,7 @@ final class CandidateQualityAudit {
|
||||
candidateCountDistribution.merge(set.size(), 1L, Math::addExact);
|
||||
for (String value : set) { inverted.computeIfAbsent(value, ignored -> new ArrayList<>()).add(index); }
|
||||
}
|
||||
final QualityResult candidateResult = CandidateAwareEvaluator.evaluate(candidate.name(), candidate.language().name(),
|
||||
final QualityResult candidateResult = CandidateAwareEvaluator.evaluate(candidate.name(), candidate.resultLanguage(),
|
||||
mode, OutputPolicy.ALL_CANDIDATES, groups, candidate.createStemmer());
|
||||
final List<Integer> selected = new ArrayList<>();
|
||||
for (int index = 0; index < forms.size(); index++) { if (candidateSets.get(index).size() > 1) { selected.add(index); } }
|
||||
@@ -92,8 +97,10 @@ final class CandidateQualityAudit {
|
||||
long repaired = 0; long introduced = 0;
|
||||
for (int partner : partners) {
|
||||
final boolean primaryRelated = primaryOutputs[index].equals(primaryOutputs[partner]);
|
||||
if (groupIndexes.get(index).equals(groupIndexes.get(partner)) && !primaryRelated) { repaired++; }
|
||||
if (!groupIndexes.get(index).equals(groupIndexes.get(partner)) && !primaryRelated) { introduced++; }
|
||||
final Set<Integer> sharedMemberships = new HashSet<>(memberships.get(index));
|
||||
sharedMemberships.retainAll(memberships.get(partner));
|
||||
if (!sharedMemberships.isEmpty() && !primaryRelated) { repaired++; }
|
||||
if (sharedMemberships.isEmpty() && !primaryRelated) { introduced++; }
|
||||
}
|
||||
words.add(new Word(rows.get(index), forms.get(index), primaryOutputs[index], candidateSets.get(index),
|
||||
repaired, introduced));
|
||||
@@ -129,8 +136,8 @@ final class CandidateQualityAudit {
|
||||
text.append("- Row ").append(word.row()).append(", form `").append(escape(word.form()))
|
||||
.append("`, primary `").append(escape(word.primary())).append("`, candidates ")
|
||||
.append(word.candidates().stream().map(value -> "`" + escape(value) + "`").toList())
|
||||
.append(", repaired same-group relations ").append(word.repairedUnderRelations())
|
||||
.append(", introduced cross-group relations ").append(word.introducedOverRelations()).append(".\n");
|
||||
.append(", repaired gold-positive relations ").append(word.repairedUnderRelations())
|
||||
.append(", introduced gold-negative relations ").append(word.introducedOverRelations()).append(".\n");
|
||||
}
|
||||
text.append('\n');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer.benchmark.quality;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Immutable overlapping gold-standard cover over unique surface forms.
|
||||
*
|
||||
* <p>A form may belong to several dictionary groups. Two distinct forms are a
|
||||
* gold-positive pair when they share at least one included group, and the pair
|
||||
* is counted once even when it shares several groups.</p>
|
||||
*/
|
||||
final class GoldStandardCover {
|
||||
private final List<GoldStandardGroup> groups;
|
||||
private final List<String> forms;
|
||||
private final List<Integer> representativeRows;
|
||||
private final Map<String, Integer> formIndexes;
|
||||
private final List<DuplicateRelation> duplicateRelations;
|
||||
private final long singletonRows;
|
||||
private final long pairRows;
|
||||
private final long relatedPairs;
|
||||
|
||||
/** Builds the cover selected by one processing mode. */
|
||||
static GoldStandardCover create(final Iterable<GoldStandardGroup> source, final ProcessingMode mode) {
|
||||
final List<GoldStandardGroup> groups = new ArrayList<>();
|
||||
final Map<String, Integer> membershipCounts = new HashMap<>();
|
||||
final LinkedHashMap<String, Integer> representativeRows = new LinkedHashMap<>();
|
||||
long singletonRows = 0;
|
||||
long pairRows = 0;
|
||||
long rawRelatedPairs = 0;
|
||||
for (GoldStandardGroup group : source) {
|
||||
if (!mode.includes(group.forms())) {
|
||||
continue;
|
||||
}
|
||||
groups.add(group);
|
||||
if (group.forms().size() == 1) {
|
||||
singletonRows = add(singletonRows, 1, "singleton dictionary rows");
|
||||
} else {
|
||||
pairRows = add(pairRows, 1, "dictionary rows contributing related pairs");
|
||||
}
|
||||
rawRelatedPairs = add(rawRelatedPairs, QualityEvaluator.chooseTwo(group.forms().size()),
|
||||
"raw gold-related pairs");
|
||||
for (String form : group.forms()) {
|
||||
membershipCounts.merge(form, 1, Math::addExact);
|
||||
representativeRows.putIfAbsent(form, group.rowNumber());
|
||||
}
|
||||
}
|
||||
|
||||
final List<String> forms = List.copyOf(representativeRows.keySet());
|
||||
final Map<String, Integer> formIndexes = new HashMap<>(forms.size() * 2);
|
||||
final List<Integer> rows = new ArrayList<>(forms.size());
|
||||
for (int index = 0; index < forms.size(); index++) {
|
||||
final String form = forms.get(index);
|
||||
formIndexes.put(form, index);
|
||||
rows.add(representativeRows.get(form));
|
||||
}
|
||||
|
||||
final Set<Long> seenRelations = new HashSet<>();
|
||||
final Map<Long, Integer> extraOccurrences = new HashMap<>();
|
||||
for (GoldStandardGroup group : groups) {
|
||||
final List<Integer> repeated = new ArrayList<>();
|
||||
for (String form : group.forms()) {
|
||||
if (membershipCounts.get(form) > 1) {
|
||||
repeated.add(formIndexes.get(form));
|
||||
}
|
||||
}
|
||||
for (int left = 0; left < repeated.size(); left++) {
|
||||
for (int right = left + 1; right < repeated.size(); right++) {
|
||||
final long key = pairKey(repeated.get(left), repeated.get(right));
|
||||
if (!seenRelations.add(key)) {
|
||||
extraOccurrences.merge(key, 1, Math::addExact);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
final List<DuplicateRelation> duplicates = new ArrayList<>(extraOccurrences.size());
|
||||
long duplicateCount = 0;
|
||||
for (Map.Entry<Long, Integer> entry : extraOccurrences.entrySet()) {
|
||||
final long key = entry.getKey();
|
||||
final int extra = entry.getValue();
|
||||
duplicates.add(new DuplicateRelation((int) (key >>> 32), (int) key, extra));
|
||||
duplicateCount = add(duplicateCount, extra, "duplicate gold-relation occurrences");
|
||||
}
|
||||
duplicates.sort(null);
|
||||
return new GoldStandardCover(List.copyOf(groups), forms, List.copyOf(rows), Map.copyOf(formIndexes),
|
||||
List.copyOf(duplicates), singletonRows, pairRows,
|
||||
subtract(rawRelatedPairs, duplicateCount, "unique gold-related pairs"));
|
||||
}
|
||||
|
||||
private GoldStandardCover(final List<GoldStandardGroup> groups, final List<String> forms,
|
||||
final List<Integer> representativeRows, final Map<String, Integer> formIndexes,
|
||||
final List<DuplicateRelation> duplicateRelations, final long singletonRows,
|
||||
final long pairRows, final long relatedPairs) {
|
||||
this.groups = groups;
|
||||
this.forms = forms;
|
||||
this.representativeRows = representativeRows;
|
||||
this.formIndexes = formIndexes;
|
||||
this.duplicateRelations = duplicateRelations;
|
||||
this.singletonRows = singletonRows;
|
||||
this.pairRows = pairRows;
|
||||
this.relatedPairs = relatedPairs;
|
||||
}
|
||||
|
||||
/** Returns the included source groups. */
|
||||
List<GoldStandardGroup> groups() { return groups; }
|
||||
/** Returns every included surface form exactly once. */
|
||||
List<String> forms() { return forms; }
|
||||
/** Returns a source row suitable for diagnostics for one unique form. */
|
||||
int representativeRow(final int formIndex) { return representativeRows.get(formIndex); }
|
||||
/** Returns the unique index of a surface form. */
|
||||
int indexOf(final String form) { return formIndexes.get(form); }
|
||||
/** Returns relations repeated by more than one group. */
|
||||
List<DuplicateRelation> duplicateRelations() { return duplicateRelations; }
|
||||
/** Returns the number of included singleton rows. */
|
||||
long singletonRows() { return singletonRows; }
|
||||
/** Returns the number of included rows containing a relation. */
|
||||
long pairRows() { return pairRows; }
|
||||
/** Returns the number of unique gold-positive form pairs. */
|
||||
long relatedPairs() { return relatedPairs; }
|
||||
|
||||
/** Encodes an unordered pair of non-negative form indexes. */
|
||||
private static long pairKey(final int first, final int second) {
|
||||
final int left = Math.min(first, second);
|
||||
final int right = Math.max(first, second);
|
||||
return ((long) left << 32) | (right & 0xffffffffL);
|
||||
}
|
||||
|
||||
/** Checked addition with metric context. */
|
||||
private static long add(final long left, final long right, final String context) {
|
||||
try {
|
||||
return Math.addExact(left, right);
|
||||
} catch (ArithmeticException exception) {
|
||||
throw new IllegalStateException("Arithmetic overflow in " + context + ".", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/** Checked subtraction with metric context. */
|
||||
private static long subtract(final long left, final long right, final String context) {
|
||||
try {
|
||||
return Math.subtractExact(left, right);
|
||||
} catch (ArithmeticException exception) {
|
||||
throw new IllegalStateException("Arithmetic overflow in " + context + ".", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/** One relation counted by more than one source group. */
|
||||
record DuplicateRelation(int leftFormIndex, int rightFormIndex, int extraOccurrences)
|
||||
implements Comparable<DuplicateRelation> {
|
||||
/** Orders relations deterministically by their form indexes. */
|
||||
@Override
|
||||
public int compareTo(final DuplicateRelation other) {
|
||||
final int leftComparison = Integer.compare(leftFormIndex, other.leftFormIndex);
|
||||
return leftComparison != 0 ? leftComparison : Integer.compare(rightFormIndex, other.rightFormIndex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,26 +37,33 @@ import java.util.OptionalDouble;
|
||||
* Undefined ratios are represented by empty optionals; no method returns NaN or infinity.
|
||||
*/
|
||||
record PairwiseMetrics(long truePositivePairs, long falsePositivePairs, long falseNegativePairs,
|
||||
long trueNegativePairs) {
|
||||
long trueNegativePairs, boolean coherentConfusionMatrix) {
|
||||
/** Creates metrics for one coherent binary relation. */
|
||||
PairwiseMetrics(final long truePositivePairs, final long falsePositivePairs,
|
||||
final long falseNegativePairs, final long trueNegativePairs) {
|
||||
this(truePositivePairs, falsePositivePairs, falseNegativePairs, trueNegativePairs, true);
|
||||
}
|
||||
|
||||
/** Creates checked confusion counts from one quality result. */
|
||||
static PairwiseMetrics from(final QualityResult result) {
|
||||
return new PairwiseMetrics(Math.subtractExact(result.underPossiblePairs(), result.underErrorPairs()),
|
||||
result.overErrorPairs(), result.underErrorPairs(),
|
||||
Math.subtractExact(result.overPossiblePairs(), result.overErrorPairs()));
|
||||
Math.subtractExact(result.overPossiblePairs(), result.overErrorPairs()),
|
||||
result.outputPolicy() != OutputPolicy.ANY_CANDIDATE);
|
||||
}
|
||||
|
||||
/** @return pairwise precision */ OptionalDouble precision() { return ratio(truePositivePairs, Math.addExact(truePositivePairs, falsePositivePairs)); }
|
||||
/** @return pairwise recall */ OptionalDouble recall() { return ratio(truePositivePairs, Math.addExact(truePositivePairs, falseNegativePairs)); }
|
||||
/** @return pairwise specificity */ OptionalDouble specificity() { return ratio(trueNegativePairs, Math.addExact(trueNegativePairs, falsePositivePairs)); }
|
||||
/** @return pairwise precision */ OptionalDouble precision() { return coherentRatio(truePositivePairs, Math.addExact(truePositivePairs, falsePositivePairs)); }
|
||||
/** @return pairwise recall */ OptionalDouble recall() { return coherentRatio(truePositivePairs, Math.addExact(truePositivePairs, falseNegativePairs)); }
|
||||
/** @return pairwise specificity */ OptionalDouble specificity() { return coherentRatio(trueNegativePairs, Math.addExact(trueNegativePairs, falsePositivePairs)); }
|
||||
/** @return pairwise accuracy, potentially dominated by true negatives */
|
||||
OptionalDouble accuracy() { return ratio(Math.addExact(truePositivePairs, trueNegativePairs), total()); }
|
||||
OptionalDouble accuracy() { return coherentRatio(Math.addExact(truePositivePairs, trueNegativePairs), total()); }
|
||||
/** @return arithmetic mean of recall and specificity */
|
||||
OptionalDouble balancedAccuracy() { return mean(recall(), specificity()); }
|
||||
/** @return pairwise F0.5 */ OptionalDouble f05() { return fBeta(0.25); }
|
||||
/** @return pairwise F1 */ OptionalDouble f1() { return fBeta(1.0); }
|
||||
/** @return pairwise F2 */ OptionalDouble f2() { return fBeta(4.0); }
|
||||
/** @return Jaccard index */
|
||||
OptionalDouble jaccard() { return ratio(truePositivePairs, Math.addExact(Math.addExact(truePositivePairs, falsePositivePairs), falseNegativePairs)); }
|
||||
OptionalDouble jaccard() { return coherentRatio(truePositivePairs, Math.addExact(Math.addExact(truePositivePairs, falsePositivePairs), falseNegativePairs)); }
|
||||
/** @return Fowlkes-Mallows index */
|
||||
OptionalDouble fowlkesMallows() {
|
||||
final OptionalDouble precisionValue = precision(); final OptionalDouble recallValue = recall();
|
||||
@@ -65,6 +72,7 @@ record PairwiseMetrics(long truePositivePairs, long falsePositivePairs, long fal
|
||||
}
|
||||
/** @return Matthews correlation coefficient using scaled double arithmetic */
|
||||
OptionalDouble matthewsCorrelationCoefficient() {
|
||||
if (!coherentConfusionMatrix) { return OptionalDouble.empty(); }
|
||||
final double a = (double) truePositivePairs + falsePositivePairs;
|
||||
final double b = (double) truePositivePairs + falseNegativePairs;
|
||||
final double c = (double) trueNegativePairs + falsePositivePairs;
|
||||
@@ -76,10 +84,11 @@ record PairwiseMetrics(long truePositivePairs, long falsePositivePairs, long fal
|
||||
return OptionalDouble.of(numerator / denominator);
|
||||
}
|
||||
/** @return pairwise error rate */
|
||||
OptionalDouble errorRate() { return ratio(Math.addExact(falsePositivePairs, falseNegativePairs), total()); }
|
||||
OptionalDouble errorRate() { return coherentRatio(Math.addExact(falsePositivePairs, falseNegativePairs), total()); }
|
||||
|
||||
/** Calculates F-beta directly from raw counts. */
|
||||
private OptionalDouble fBeta(final double betaSquared) {
|
||||
if (!coherentConfusionMatrix) { return OptionalDouble.empty(); }
|
||||
final double numerator = (1.0 + betaSquared) * truePositivePairs;
|
||||
final double denominator = numerator + betaSquared * falseNegativePairs + falsePositivePairs;
|
||||
return denominator == 0.0 ? OptionalDouble.empty() : OptionalDouble.of(numerator / denominator);
|
||||
@@ -90,6 +99,10 @@ record PairwiseMetrics(long truePositivePairs, long falsePositivePairs, long fal
|
||||
private static OptionalDouble ratio(final long numerator, final long denominator) {
|
||||
return denominator == 0 ? OptionalDouble.empty() : OptionalDouble.of((double) numerator / denominator);
|
||||
}
|
||||
/** Calculates a ratio only when the counts describe one coherent relation. */
|
||||
private OptionalDouble coherentRatio(final long numerator, final long denominator) {
|
||||
return coherentConfusionMatrix ? ratio(numerator, denominator) : OptionalDouble.empty();
|
||||
}
|
||||
/** Averages two defined ratios. */
|
||||
private static OptionalDouble mean(final OptionalDouble left, final OptionalDouble right) {
|
||||
return left.isEmpty() || right.isEmpty() ? OptionalDouble.empty()
|
||||
|
||||
@@ -70,4 +70,14 @@ final class PairwiseMetricsTest {
|
||||
assertTrue(metrics.precision().isEmpty()); assertTrue(metrics.recall().isEmpty());
|
||||
assertTrue(metrics.matthewsCorrelationCoefficient().isEmpty());
|
||||
}
|
||||
|
||||
/** Verifies oracle-assisted bounds are not misreported as one confusion matrix. */
|
||||
@Test @DisplayName("Oracle-assisted ANY policy suppresses classification aggregates")
|
||||
void oracleAssistedPolicy() {
|
||||
final PairwiseMetrics metrics = new PairwiseMetrics(10, 2, 3, 20, false);
|
||||
assertTrue(metrics.precision().isEmpty());
|
||||
assertTrue(metrics.recall().isEmpty());
|
||||
assertTrue(metrics.f1().isEmpty());
|
||||
assertTrue(metrics.matthewsCorrelationCoefficient().isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,28 +63,30 @@ final class QualityAudit {
|
||||
static Scenario evaluate(final Candidate candidate, final ProcessingMode mode,
|
||||
final List<GoldStandardGroup> groups, final int limit) throws IOException {
|
||||
final List<GoldStandardGroup> includedGroups = groups.stream().filter(group -> mode.includes(group.forms())).toList();
|
||||
final List<String> forms = new ArrayList<>();
|
||||
for (GoldStandardGroup group : includedGroups) {
|
||||
forms.addAll(group.forms());
|
||||
}
|
||||
final GoldStandardCover cover = GoldStandardCover.create(groups, mode);
|
||||
final List<String> forms = cover.forms();
|
||||
final String[] outputs = candidate.createStemmer().stem(forms.toArray(String[]::new));
|
||||
if (outputs.length != forms.size()) {
|
||||
throw new IOException("Invalid audit output count for stemmer " + candidate.name() + ", language "
|
||||
+ candidate.language() + ", and processing mode " + mode + ".");
|
||||
+ candidate.resultLanguage() + ", and processing mode " + mode + ".");
|
||||
}
|
||||
final int[] outputIndex = {0};
|
||||
final QualityResult result = QualityEvaluator.evaluate(candidate.name(), candidate.language().name(), mode,
|
||||
groups, word -> outputs[outputIndex[0]++]);
|
||||
final Map<String, String> outputsByForm = new LinkedHashMap<>();
|
||||
for (int index = 0; index < forms.size(); index++) {
|
||||
outputsByForm.put(forms.get(index), outputs[index]);
|
||||
}
|
||||
final QualityResult result = QualityEvaluator.evaluate(candidate.name(), candidate.resultLanguage(), mode,
|
||||
groups, outputsByForm::get);
|
||||
final List<Contributor> contributors = new ArrayList<>();
|
||||
long exactMatches = 0;
|
||||
int offset = 0;
|
||||
long exactDenominator = 0;
|
||||
final List<Integer> sizes = new ArrayList<>();
|
||||
for (GoldStandardGroup group : includedGroups) {
|
||||
final Map<String, List<String>> formsByStem = new LinkedHashMap<>();
|
||||
final String expected = group.forms().get(0);
|
||||
long mergedPairs = 0;
|
||||
for (String form : group.forms()) {
|
||||
final String output = outputs[offset++];
|
||||
exactDenominator++;
|
||||
final String output = outputsByForm.get(form);
|
||||
formsByStem.computeIfAbsent(output, ignored -> new ArrayList<>()).add(form);
|
||||
if (expected.equals(output)) {
|
||||
exactMatches++;
|
||||
@@ -103,16 +105,12 @@ final class QualityAudit {
|
||||
contributors.sort(Comparator.comparingLong(Contributor::errorPairs).reversed()
|
||||
.thenComparingInt(Contributor::rowNumber));
|
||||
final long contributionSum = contributors.stream().mapToLong(Contributor::errorPairs).reduce(0L, Math::addExact);
|
||||
if (contributionSum != result.underErrorPairs()) {
|
||||
throw new IOException("The summed group contributions do not equal the optimized under-stemming total for "
|
||||
+ candidate.name() + ", " + candidate.language() + ", and " + mode + ".");
|
||||
}
|
||||
sizes.sort(Integer::compareTo);
|
||||
final double mean = sizes.stream().mapToInt(Integer::intValue).average().orElse(0.0);
|
||||
final double median = median(sizes);
|
||||
final String resource = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
|
||||
.requireDefault(candidate.language()).resource();
|
||||
return new Scenario(result, resource, exactMatches, forms.size(),
|
||||
.require(candidate.dictionaryModelId()).resource();
|
||||
return new Scenario(result, resource, exactMatches, exactDenominator,
|
||||
sizes.isEmpty() ? 0 : sizes.get(0), sizes.isEmpty() ? 0 : sizes.get(sizes.size() - 1), mean, median,
|
||||
List.copyOf(contributors.subList(0, Math.min(limit, contributors.size()))), contributionSum);
|
||||
}
|
||||
@@ -136,7 +134,8 @@ final class QualityAudit {
|
||||
.append("- Exact first-field matches: ").append(scenario.exactMatches()).append(" / ").append(scenario.exactDenominator()).append("\n")
|
||||
.append("- Under-stemming pairs: ").append(result.underErrorPairs()).append(" / ").append(result.underPossiblePairs()).append("\n")
|
||||
.append("- Over-stemming pairs: ").append(result.overErrorPairs()).append(" / ").append(result.overPossiblePairs()).append("\n")
|
||||
.append("- Independently summed under-stemming contributions: ").append(scenario.contributionSum()).append("\n\n")
|
||||
.append("- Sum of row-local under-stemming contributions: ").append(scenario.contributionSum())
|
||||
.append(" (shared gold pairs can occur in more than one row)\n\n")
|
||||
.append("### Highest under-stemming contributors\n\n");
|
||||
for (Contributor contributor : scenario.contributors()) {
|
||||
text.append("#### Dictionary row ").append(contributor.rowNumber()).append("\n\n")
|
||||
|
||||
@@ -33,15 +33,14 @@ package org.egothor.stemmer.benchmark.quality;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.egothor.stemmer.benchmark.QualityStemmerMatrix.BatchStemmer;
|
||||
|
||||
/** Evaluates pairwise partition agreement using aggregated frequencies, never explicit pairs. */
|
||||
/** Evaluates pairwise agreement with an overlapping gold-standard cover. */
|
||||
public final class QualityEvaluator {
|
||||
/** Utility class. */
|
||||
private QualityEvaluator() { throw new AssertionError("No instances."); }
|
||||
@@ -61,53 +60,52 @@ public final class QualityEvaluator {
|
||||
final Iterable<GoldStandardGroup> groups, final StemmerFunction stemmer) {
|
||||
Objects.requireNonNull(groups, "groups");
|
||||
Objects.requireNonNull(stemmer, "stemmer");
|
||||
return evaluate(stemmerName, language, mode, GoldStandardCover.create(groups, mode), stemmer);
|
||||
}
|
||||
|
||||
/** Evaluates one scenario over a prebuilt overlapping gold-standard cover. */
|
||||
private static QualityResult evaluate(final String stemmerName, final String language,
|
||||
final ProcessingMode mode, final GoldStandardCover cover, final StemmerFunction stemmer) {
|
||||
final Map<String, Long> global = new HashMap<>();
|
||||
long rows = 0;
|
||||
long words = 0;
|
||||
long singletonRows = 0;
|
||||
long pairRows = 0;
|
||||
long underPossible = 0;
|
||||
long withinSameStem = 0;
|
||||
final Set<String> stems = new HashSet<>();
|
||||
final Map<String, Long> local = new HashMap<>();
|
||||
final List<Map<String, Long>> contingency = new ArrayList<>();
|
||||
final List<Long> groupSizes = new ArrayList<>();
|
||||
for (GoldStandardGroup group : groups) {
|
||||
final List<String> forms = group.forms();
|
||||
if (!mode.includes(forms)) {
|
||||
continue;
|
||||
final String[] outputs = new String[cover.forms().size()];
|
||||
for (int index = 0; index < outputs.length; index++) {
|
||||
final String form = cover.forms().get(index);
|
||||
final String output;
|
||||
try {
|
||||
output = stemmer.stem(form);
|
||||
} catch (IOException exception) {
|
||||
throw failure(stemmerName, language, mode, cover.representativeRow(index), form,
|
||||
"the stemmer threw an exception", exception);
|
||||
}
|
||||
rows = add(rows, 1, "applied dictionary rows");
|
||||
words = add(words, forms.size(), "processed word forms");
|
||||
if (forms.size() == 1) {
|
||||
singletonRows = add(singletonRows, 1, "singleton dictionary rows");
|
||||
} else {
|
||||
pairRows = add(pairRows, 1, "dictionary rows contributing under-stemming pairs");
|
||||
if (output == null) {
|
||||
throw failure(stemmerName, language, mode, cover.representativeRow(index), form,
|
||||
"the stemmer returned null", null);
|
||||
}
|
||||
underPossible = add(underPossible, chooseTwo(forms.size()), "under-stemming possible pairs");
|
||||
outputs[index] = output;
|
||||
global.merge(output, 1L, (left, right) -> add(left, right, "global stem frequency"));
|
||||
stems.add(output);
|
||||
}
|
||||
for (GoldStandardGroup group : cover.groups()) {
|
||||
local.clear();
|
||||
for (String form : forms) {
|
||||
final String output;
|
||||
try {
|
||||
output = stemmer.stem(form);
|
||||
} catch (IOException exception) {
|
||||
throw failure(stemmerName, language, mode, group.rowNumber(), form,
|
||||
"the stemmer threw an exception", exception);
|
||||
}
|
||||
if (output == null) {
|
||||
throw failure(stemmerName, language, mode, group.rowNumber(), form,
|
||||
"the stemmer returned null", null);
|
||||
}
|
||||
for (String form : group.forms()) {
|
||||
final String output = outputs[cover.indexOf(form)];
|
||||
local.merge(output, 1L, (left, right) -> add(left, right, "group-to-stem frequency"));
|
||||
global.merge(output, 1L, (left, right) -> add(left, right, "global stem frequency"));
|
||||
stems.add(output);
|
||||
}
|
||||
for (long frequency : local.values()) {
|
||||
withinSameStem = add(withinSameStem, chooseTwo(frequency), "within-group merged pairs");
|
||||
}
|
||||
contingency.add(Map.copyOf(local));
|
||||
groupSizes.add((long) forms.size());
|
||||
}
|
||||
for (GoldStandardCover.DuplicateRelation duplicate : cover.duplicateRelations()) {
|
||||
if (outputs[duplicate.leftFormIndex()].equals(outputs[duplicate.rightFormIndex()])) {
|
||||
withinSameStem = subtract(withinSameStem, duplicate.extraOccurrences(),
|
||||
"duplicate within-group merged pairs");
|
||||
}
|
||||
}
|
||||
final long words = cover.forms().size();
|
||||
final long underPossible = cover.relatedPairs();
|
||||
long allPairs = chooseTwo(words);
|
||||
long overPossible = subtract(allPairs, underPossible, "over-stemming possible pairs");
|
||||
long allSameStem = 0;
|
||||
@@ -116,47 +114,10 @@ public final class QualityEvaluator {
|
||||
}
|
||||
final long underError = subtract(underPossible, withinSameStem, "under-stemming error pairs");
|
||||
final long overError = subtract(allSameStem, withinSameStem, "over-stemming error pairs");
|
||||
final PartitionMetrics partition = partitionMetrics(words, underPossible, allSameStem,
|
||||
withinSameStem, groupSizes, global, contingency);
|
||||
return new QualityResult(stemmerName, language, mode, OutputPolicy.PRIMARY_OUTPUT,
|
||||
rows, words, singletonRows, pairRows, words, 0, words == 0 ? 0 : 1, words, stems.size(), overError, overPossible,
|
||||
underError, underPossible, partition);
|
||||
}
|
||||
|
||||
/** Calculates strict-partition metrics from the exact contingency table. */
|
||||
private static PartitionMetrics partitionMetrics(final long words, final long rowPairs, final long columnPairs,
|
||||
final long indexPairs, final List<Long> groupSizes, final Map<String, Long> global,
|
||||
final List<Map<String, Long>> contingency) {
|
||||
if (words == 0) { return new PartitionMetrics(0.0, 0.0, 0.0, 0.0, 0.0); }
|
||||
final double totalPairs = chooseTwo(words);
|
||||
final double expected = totalPairs == 0.0 ? 0.0 : (double) rowPairs * columnPairs / totalPairs;
|
||||
final double maximum = (rowPairs + (double) columnPairs) / 2.0;
|
||||
final double adjustedRand = maximum == expected ? 1.0 : (indexPairs - expected) / (maximum - expected);
|
||||
final double goldEntropy = entropy(words, groupSizes);
|
||||
final double predictedEntropy = entropy(words, global.values());
|
||||
double mutualInformation = 0.0;
|
||||
for (int group = 0; group < contingency.size(); group++) {
|
||||
final long groupSize = groupSizes.get(group);
|
||||
for (Map.Entry<String, Long> cell : contingency.get(group).entrySet()) {
|
||||
final double frequency = cell.getValue();
|
||||
mutualInformation += frequency / words * Math.log(frequency * words
|
||||
/ (groupSize * (double) global.get(cell.getKey())));
|
||||
}
|
||||
}
|
||||
final double homogeneity = goldEntropy == 0.0 ? 1.0 : mutualInformation / goldEntropy;
|
||||
final double completeness = predictedEntropy == 0.0 ? 1.0 : mutualInformation / predictedEntropy;
|
||||
final double vMeasure = homogeneity + completeness == 0.0 ? 0.0
|
||||
: 2.0 * homogeneity * completeness / (homogeneity + completeness);
|
||||
final double nmiDenominator = (goldEntropy + predictedEntropy) / 2.0;
|
||||
final double nmi = nmiDenominator == 0.0 ? 1.0 : mutualInformation / nmiDenominator;
|
||||
return new PartitionMetrics(adjustedRand, homogeneity, completeness, vMeasure, nmi);
|
||||
}
|
||||
|
||||
/** Calculates natural-log entropy from category frequencies. */
|
||||
private static double entropy(final long total, final Iterable<Long> frequencies) {
|
||||
double weightedLogs = 0.0;
|
||||
for (long frequency : frequencies) { weightedLogs += frequency * Math.log(frequency); }
|
||||
return Math.log(total) - weightedLogs / total;
|
||||
cover.groups().size(), words, cover.singletonRows(), cover.pairRows(), words, 0,
|
||||
words == 0 ? 0 : 1, words, stems.size(), overError, overPossible,
|
||||
underError, underPossible, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -175,19 +136,14 @@ public final class QualityEvaluator {
|
||||
public static QualityResult evaluateBatch(final String stemmerName, final String language,
|
||||
final ProcessingMode mode, final List<GoldStandardGroup> groups, final BatchStemmer stemmer)
|
||||
throws IOException {
|
||||
final List<String> included = new ArrayList<>();
|
||||
for (GoldStandardGroup group : groups) {
|
||||
if (mode.includes(group.forms())) {
|
||||
included.addAll(group.forms());
|
||||
}
|
||||
}
|
||||
final String[] outputs = stemmer.stem(included.toArray(String[]::new));
|
||||
if (outputs == null || outputs.length != included.size()) {
|
||||
final GoldStandardCover cover = GoldStandardCover.create(groups, mode);
|
||||
final String[] outputs = stemmer.stem(cover.forms().toArray(String[]::new));
|
||||
if (outputs == null || outputs.length != cover.forms().size()) {
|
||||
throw new IOException("JMH stemmer " + stemmerName + " returned an invalid output batch for language "
|
||||
+ language + " and processing mode " + mode + ".");
|
||||
}
|
||||
final int[] index = {0};
|
||||
return evaluate(stemmerName, language, mode, groups, word -> {
|
||||
return evaluate(stemmerName, language, mode, cover, word -> {
|
||||
final String output = outputs[index[0]++];
|
||||
if (output == null) {
|
||||
throw new IOException("JMH stemmer " + stemmerName + " returned null for language " + language
|
||||
|
||||
@@ -32,6 +32,7 @@ package org.egothor.stemmer.benchmark.quality;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@@ -58,11 +59,7 @@ final class QualityEvaluatorTest {
|
||||
assertEquals(0, result.overErrorPairs()); assertEquals(4, result.overPossiblePairs());
|
||||
assertEquals(0, result.underErrorPairs()); assertEquals(2, result.underPossiblePairs());
|
||||
assertEquals(2, result.distinctOutputStems());
|
||||
assertEquals(1.0, result.partitionMetrics().adjustedRandIndex(), 1.0e-12);
|
||||
assertEquals(1.0, result.partitionMetrics().homogeneity(), 1.0e-12);
|
||||
assertEquals(1.0, result.partitionMetrics().completeness(), 1.0e-12);
|
||||
assertEquals(1.0, result.partitionMetrics().vMeasure(), 1.0e-12);
|
||||
assertEquals(1.0, result.partitionMetrics().normalizedMutualInformation(), 1.0e-12);
|
||||
assertNull(result.partitionMetrics());
|
||||
}
|
||||
/** Verifies partial merge and pure under-stemming pair counts. */
|
||||
@Test @DisplayName("A partial within-group merge is counted by pairs")
|
||||
@@ -88,13 +85,35 @@ final class QualityEvaluatorTest {
|
||||
assertEquals(2, result.overErrorPairs()); assertEquals(6, result.overPossiblePairs());
|
||||
assertEquals(2, result.underErrorPairs()); assertEquals(4, result.underPossiblePairs());
|
||||
}
|
||||
/** Verifies duplicate scope and singleton undefined denominator. */
|
||||
@Test @DisplayName("Duplicates are removed only within a group and singleton under-stemming is undefined")
|
||||
/** Verifies unique-form identity across overlapping groups. */
|
||||
@Test @DisplayName("The same form in several groups remains one corpus item")
|
||||
void duplicateScope() {
|
||||
final QualityResult result = evaluate(List.of(group(1, "same", "same"), group(2, "same")), Map.of("same", "x"));
|
||||
assertEquals(2, result.processedWordForms()); assertEquals(1, result.overErrorPairs());
|
||||
assertEquals(1, result.processedWordForms()); assertEquals(0, result.overErrorPairs());
|
||||
assertTrue(result.underPercentage().isEmpty());
|
||||
}
|
||||
|
||||
/** Verifies a form can participate in several gold relations without duplication. */
|
||||
@Test @DisplayName("Overlapping group memberships define a deduplicated gold relation")
|
||||
void overlappingMemberships() {
|
||||
final QualityResult result = evaluate(List.of(group(1, "a", "x"), group(2, "a", "y")),
|
||||
Map.of("a", "s", "x", "s", "y", "t"));
|
||||
assertEquals(3, result.processedWordForms());
|
||||
assertEquals(2, result.underPossiblePairs());
|
||||
assertEquals(1, result.underErrorPairs());
|
||||
assertEquals(1, result.overPossiblePairs());
|
||||
assertEquals(0, result.overErrorPairs());
|
||||
}
|
||||
|
||||
/** Verifies a pair shared by several groups is counted only once. */
|
||||
@Test @DisplayName("A relation shared by several groups is counted once")
|
||||
void duplicateRelation() {
|
||||
final QualityResult result = evaluate(List.of(group(1, "a", "b"), group(2, "a", "b", "c")),
|
||||
Map.of("a", "s", "b", "s", "c", "t"));
|
||||
assertEquals(3, result.underPossiblePairs());
|
||||
assertEquals(2, result.underErrorPairs());
|
||||
assertEquals(0, result.overPossiblePairs());
|
||||
}
|
||||
/** Verifies the zero over-stemming denominator. */
|
||||
@Test @DisplayName("One gold group has an undefined over-stemming percentage")
|
||||
void zeroOverDenominator() {
|
||||
|
||||
@@ -67,7 +67,7 @@ public final class QualityReportWriter {
|
||||
if (filtered) {
|
||||
text.append("> This is a filtered analytical report and is not the complete JMH candidate matrix.\n\n");
|
||||
}
|
||||
text.append("## Methodology\n\nEach parsed multilingual dictionary row is a gold-standard equivalence class. Exact duplicates are removed only within that row. `PRIMARY_OUTPUT` is the deterministic JMH partition. `ANY_CANDIDATE` is an optimistic oracle-assisted pairwise upper bound: within-row sets must intersect, while a cross-row error occurs only for two equal singleton sets. `ALL_CANDIDATES` activates the complete overlap relation: within-row disjoint sets are false negatives and cross-row intersections are false positives. A shared pair is counted once. Candidate policies need not define partitions.\n\nTP is a related within-row pair, FN is an unrelated within-row pair, FP is a related cross-row pair, and TN is an unrelated cross-row pair. Under-stemming is FN/(TP+FN); over-stemming is FP/(TN+FP), so their denominators differ. F0.5 emphasizes precision, F1 balances precision and recall, and F2 emphasizes recall. Undefined values are `n/a`. Percentages and scores use `Locale.ROOT`.\n\n| Stemmer | Language | Dictionary mode | Output policy | Applied dictionary rows | Processed word forms | Distinct output stems | Over-stemming | Under-stemming | Pairwise F0.5 | Pairwise F1 | Pairwise F2 |\n|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|\n");
|
||||
text.append("## Methodology\n\nEach distinct surface form is one evaluated item and may belong to several parsed dictionary groups. Two forms are gold-related when their membership sets intersect; a pair sharing several groups is counted once. `PRIMARY_OUTPUT` uses equality of deterministic JMH outputs. `ANY_CANDIDATE` is an optimistic oracle-assisted bound: a gold-related pair succeeds when candidate sets intersect, while a gold-negative error is unavoidable only for two equal singleton sets. `ALL_CANDIDATES` activates the complete candidate-intersection relation.\n\nUnder-stemming is the Paice Understemming Index `FN/(TP+FN)` and over-stemming is the Paice Overstemming Index `FP/(TN+FP)`, generalized here from a disjoint lemma partition to the documented overlapping gold relation. F0.5, F1, MCC, and other classification metrics require one coherent predicted relation and are therefore `n/a` for `ANY_CANDIDATE`. Standard partition metrics are not calculated because the gold memberships overlap. Undefined values are `n/a`. Percentages and scores use `Locale.ROOT`.\n\n| Stemmer | Language | Dictionary mode | Output policy | Applied dictionary rows | Processed word forms | Distinct output stems | Over-stemming | Under-stemming | Pairwise F0.5 | Pairwise F1 | Pairwise F2 |\n|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|\n");
|
||||
for (QualityResult row : rows) {
|
||||
text.append("| ").append(escapeMarkdown(row.stemmer())).append(TABLE_DELIMITER)
|
||||
.append(escapeMarkdown(row.language())).append(TABLE_DELIMITER).append(row.processingMode()).append(TABLE_DELIMITER)
|
||||
@@ -92,10 +92,12 @@ public final class QualityReportWriter {
|
||||
/** Writes machine-readable counts and separate percentage fields. */
|
||||
public static void writeCsv(final Path path, final Iterable<QualityResult> input) throws IOException {
|
||||
final StringBuilder text = new StringBuilder(4096);
|
||||
text.append("Stemmer,Language,Dictionary mode,Output policy,Applied dictionary rows,Processed word forms,Singleton dictionary rows,Forms with one candidate,Forms with multiple candidates,Maximum candidates for one form,Total candidate assignments,Distinct output stems,True-positive pairs,False-positive pairs,False-negative pairs,True-negative pairs,Over-stemming error pairs,Over-stemming possible pairs,Over-stemming percentage,Under-stemming error pairs,Under-stemming possible pairs,Under-stemming percentage,Pairwise precision,Pairwise recall,Pairwise specificity,Pairwise accuracy,Balanced accuracy,Pairwise F0.5,Pairwise F1,Pairwise F2,Jaccard index,Fowlkes-Mallows index,Matthews correlation coefficient,Pairwise error rate,Adjusted Rand Index,Homogeneity,Completeness,V-measure,Normalized mutual information\n");
|
||||
text.append("Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model SHA-256,Dictionary mode,Output policy,Applied dictionary rows,Processed word forms,Singleton dictionary rows,Forms with one candidate,Forms with multiple candidates,Maximum candidates for one form,Total candidate assignments,Distinct output stems,True-positive pairs,False-positive pairs,False-negative pairs,True-negative pairs,Over-stemming error pairs,Over-stemming possible pairs,Over-stemming percentage,Under-stemming error pairs,Under-stemming possible pairs,Under-stemming percentage,Pairwise precision,Pairwise recall,Pairwise specificity,Pairwise accuracy,Balanced accuracy,Pairwise F0.5,Pairwise F1,Pairwise F2,Jaccard index,Fowlkes-Mallows index,Matthews correlation coefficient,Pairwise error rate,Adjusted Rand Index,Homogeneity,Completeness,V-measure,Normalized mutual information\n");
|
||||
for (QualityResult row : sorted(input)) {
|
||||
final PairwiseMetrics metrics = row.pairwiseMetrics();
|
||||
appendCsv(text, row.stemmer()); appendCsv(text, row.language()); appendCsv(text, row.processingMode().name());
|
||||
appendCsv(text, row.stemmer()); appendCsv(text, row.language());
|
||||
appendCsv(text, row.dictionaryModelId()); appendCsv(text, row.dictionaryModelVersion());
|
||||
appendCsv(text, row.dictionaryModelSha256()); appendCsv(text, row.processingMode().name());
|
||||
appendCsv(text, row.outputPolicy().name());
|
||||
appendCsv(text, Long.toString(row.appliedDictionaryRows())); appendCsv(text, Long.toString(row.processedWordForms()));
|
||||
appendCsv(text, Long.toString(row.singletonDictionaryRows()));
|
||||
@@ -104,8 +106,11 @@ public final class QualityReportWriter {
|
||||
appendCsv(text, Long.toString(row.maximumCandidatesForOneWord()));
|
||||
appendCsv(text, Long.toString(row.totalCandidateAssignments()));
|
||||
appendCsv(text, Long.toString(row.distinctOutputStems()));
|
||||
appendCsv(text, Long.toString(metrics.truePositivePairs())); appendCsv(text, Long.toString(metrics.falsePositivePairs()));
|
||||
appendCsv(text, Long.toString(metrics.falseNegativePairs())); appendCsv(text, Long.toString(metrics.trueNegativePairs()));
|
||||
final boolean confusionMatrix = row.outputPolicy() != OutputPolicy.ANY_CANDIDATE;
|
||||
appendCsv(text, confusionMatrix ? Long.toString(metrics.truePositivePairs()) : "");
|
||||
appendCsv(text, confusionMatrix ? Long.toString(metrics.falsePositivePairs()) : "");
|
||||
appendCsv(text, confusionMatrix ? Long.toString(metrics.falseNegativePairs()) : "");
|
||||
appendCsv(text, confusionMatrix ? Long.toString(metrics.trueNegativePairs()) : "");
|
||||
appendCsv(text, Long.toString(row.overErrorPairs()));
|
||||
appendCsv(text, Long.toString(row.overPossiblePairs())); appendCsv(text, machinePercent(row.overPercentage()));
|
||||
appendCsv(text, Long.toString(row.underErrorPairs())); appendCsv(text, Long.toString(row.underPossiblePairs()));
|
||||
@@ -170,7 +175,7 @@ public final class QualityReportWriter {
|
||||
.append("- Actual result rows: ").append(actualRows).append("\n\n")
|
||||
.append("Unsupported third-party combinations are excluded because their authoritative JMH adapter metadata declares no mapping for that language. They are not emitted as zero-valued rows. Radixor is independently registered for every reconciled dictionary language.\n");
|
||||
final java.util.Map<String, Set<String>> support = new java.util.TreeMap<>();
|
||||
for (Candidate candidate : candidates) { support.computeIfAbsent(candidate.name(), ignored -> new TreeSet<>()).add(candidate.language().name()); }
|
||||
for (Candidate candidate : candidates) { support.computeIfAbsent(candidate.name(), ignored -> new TreeSet<>()).add(candidate.resultLanguage()); }
|
||||
text.append("\n| Adapter | Supported language count | Supported languages |\n|---|---:|---|\n");
|
||||
support.forEach((name, languages) -> text.append("| ").append(escapeMarkdown(name)).append(TABLE_DELIMITER)
|
||||
.append(languages.size()).append(TABLE_DELIMITER).append(languages).append(" |\n"));
|
||||
@@ -230,7 +235,8 @@ public final class QualityReportWriter {
|
||||
if (metrics.f1().isPresent()) { macroF1 += metrics.f1().getAsDouble(); macroCount++; }
|
||||
languages.add(row.language());
|
||||
}
|
||||
final PairwiseMetrics micro = new PairwiseMetrics(tp, fp, fn, tn);
|
||||
final PairwiseMetrics micro = new PairwiseMetrics(tp, fp, fn, tn,
|
||||
first.outputPolicy() != OutputPolicy.ANY_CANDIDATE);
|
||||
text.append("| ").append(escapeMarkdown(first.stemmer())).append(TABLE_DELIMITER).append(first.processingMode())
|
||||
.append(TABLE_DELIMITER).append(first.outputPolicy()).append(TABLE_DELIMITER).append(languages.size())
|
||||
.append(TABLE_DELIMITER).append(score(micro.f05())).append(TABLE_DELIMITER).append(score(micro.f1()))
|
||||
|
||||
@@ -71,7 +71,7 @@ final class QualityReportWriterTest {
|
||||
final Path report = this.temporaryDirectory.resolve("report.csv");
|
||||
QualityReportWriter.writeCsv(report, List.of(result("Stemmer, \"quoted\"", "A", 0, 0)));
|
||||
final String text = Files.readString(report, StandardCharsets.UTF_8);
|
||||
assertTrue(text.startsWith("Stemmer,Language,Dictionary mode,Output policy,Applied dictionary rows,Processed word forms,Singleton dictionary rows,Forms with one candidate,"));
|
||||
assertTrue(text.startsWith("Stemmer,Language,Dictionary model ID,Dictionary model version,Dictionary model SHA-256,Dictionary mode,Output policy,Applied dictionary rows,Processed word forms,Singleton dictionary rows,Forms with one candidate,"));
|
||||
assertTrue(text.contains("\"Stemmer, \"\"quoted\"\"\""));
|
||||
assertTrue(text.contains("Adjusted Rand Index,Homogeneity,Completeness,V-measure,Normalized mutual information"));
|
||||
}
|
||||
|
||||
@@ -35,7 +35,9 @@ import java.util.Objects;
|
||||
import java.util.OptionalDouble;
|
||||
|
||||
/** Immutable pairwise stemming-quality result; all pair quantities are counts. */
|
||||
public record QualityResult(String stemmer, String language, ProcessingMode processingMode,
|
||||
public record QualityResult(String stemmer, String language,
|
||||
String dictionaryModelId, String dictionaryModelVersion, String dictionaryModelSha256,
|
||||
ProcessingMode processingMode,
|
||||
OutputPolicy outputPolicy,
|
||||
long appliedDictionaryRows, long processedWordForms, long singletonDictionaryRows,
|
||||
long dictionaryRowsContributingUnderPairs, long formsWithOneCandidate, long formsWithMultipleCandidates,
|
||||
@@ -51,6 +53,9 @@ public record QualityResult(String stemmer, String language, ProcessingMode proc
|
||||
public QualityResult {
|
||||
Objects.requireNonNull(stemmer, "stemmer");
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(dictionaryModelId, "dictionaryModelId");
|
||||
Objects.requireNonNull(dictionaryModelVersion, "dictionaryModelVersion");
|
||||
Objects.requireNonNull(dictionaryModelSha256, "dictionaryModelSha256");
|
||||
Objects.requireNonNull(processingMode, "processingMode");
|
||||
Objects.requireNonNull(outputPolicy, "outputPolicy");
|
||||
final long[] counts = {appliedDictionaryRows, processedWordForms, singletonDictionaryRows,
|
||||
@@ -70,6 +75,32 @@ public record QualityResult(String stemmer, String language, ProcessingMode proc
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an evaluator result before model provenance is attached by the application. */
|
||||
public QualityResult(final String stemmer, final String language, final ProcessingMode processingMode,
|
||||
final OutputPolicy outputPolicy, final long appliedDictionaryRows, final long processedWordForms,
|
||||
final long singletonDictionaryRows, final long dictionaryRowsContributingUnderPairs,
|
||||
final long formsWithOneCandidate, final long formsWithMultipleCandidates,
|
||||
final long maximumCandidatesForOneWord, final long totalCandidateAssignments,
|
||||
final long distinctOutputStems, final long overErrorPairs, final long overPossiblePairs,
|
||||
final long underErrorPairs, final long underPossiblePairs, final PartitionMetrics partitionMetrics) {
|
||||
this(stemmer, language, "", "", "", processingMode, outputPolicy, appliedDictionaryRows,
|
||||
processedWordForms, singletonDictionaryRows, dictionaryRowsContributingUnderPairs,
|
||||
formsWithOneCandidate, formsWithMultipleCandidates, maximumCandidatesForOneWord,
|
||||
totalCandidateAssignments, distinctOutputStems, overErrorPairs, overPossiblePairs,
|
||||
underErrorPairs, underPossiblePairs, partitionMetrics);
|
||||
}
|
||||
|
||||
/** Returns this result with immutable dictionary-model provenance attached. */
|
||||
public QualityResult withModelProvenance(final String modelId, final String modelVersion,
|
||||
final String modelSha256) {
|
||||
return new QualityResult(stemmer, language, modelId, modelVersion, modelSha256,
|
||||
processingMode, outputPolicy, appliedDictionaryRows, processedWordForms,
|
||||
singletonDictionaryRows, dictionaryRowsContributingUnderPairs, formsWithOneCandidate,
|
||||
formsWithMultipleCandidates, maximumCandidatesForOneWord, totalCandidateAssignments,
|
||||
distinctOutputStems, overErrorPairs, overPossiblePairs, underErrorPairs,
|
||||
underPossiblePairs, partitionMetrics);
|
||||
}
|
||||
|
||||
/** @return over-stemming percentage, or empty when its denominator is zero */
|
||||
public OptionalDouble overPercentage() { return percentage(overErrorPairs, overPossiblePairs); }
|
||||
/** @return under-stemming percentage, or empty when its denominator is zero */
|
||||
|
||||
@@ -37,8 +37,10 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
import org.egothor.stemmer.benchmark.QualityStemmerMatrix;
|
||||
import org.egothor.stemmer.benchmark.QualityStemmerMatrix.Candidate;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
@@ -57,10 +59,26 @@ final class QualityStemmerMatrixTest {
|
||||
@Test @DisplayName("Candidate discovery is derived from every JMH quality candidate")
|
||||
void discoversEveryCandidate() {
|
||||
final List<Candidate> candidates = QualityStemmerMatrix.candidates();
|
||||
assertEquals(92, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed.");
|
||||
assertEquals(98, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed.");
|
||||
assertTrue(candidates.stream().anyMatch(candidate -> !candidate.name().endsWith("_RADIXOR")));
|
||||
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("DA_DK_RADIXOR")));
|
||||
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("YI_RADIXOR")));
|
||||
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_POLIMORF_RADIXOR")
|
||||
&& candidate.resultLanguage().equals("pl-pl-polimorf")));
|
||||
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_LUCENE_STEMPEL_DIRECT")
|
||||
&& candidate.resultLanguage().equals("pl-pl-polimorf")));
|
||||
}
|
||||
|
||||
/** Verifies the publishable complete matrix uses only registered defaults. */
|
||||
@Test @DisplayName("Complete publication selection excludes optional models")
|
||||
void completePublicationSelectionUsesOnlyDefaultModels() {
|
||||
final List<Candidate> candidates = StemmingQualityApplication.selectCandidates(
|
||||
EnumSet.allOf(Language.class), "");
|
||||
assertEquals(92, candidates.size());
|
||||
assertTrue(candidates.stream().allMatch(candidate ->
|
||||
candidate.dictionaryModelId().equals(candidate.language().defaultModelId())));
|
||||
assertTrue(candidates.stream().noneMatch(candidate ->
|
||||
candidate.dictionaryModelId().equals("pl-pl-polimorf")));
|
||||
}
|
||||
|
||||
/** Verifies a complete report row exists for both modes of every discovered candidate. */
|
||||
@@ -69,7 +87,7 @@ final class QualityStemmerMatrixTest {
|
||||
final List<QualityResult> rows = new ArrayList<>();
|
||||
for (Candidate candidate : QualityStemmerMatrix.candidates()) {
|
||||
for (ProcessingMode mode : ProcessingMode.values()) {
|
||||
rows.add(new QualityResult(candidate.name(), candidate.language().name(), mode,
|
||||
rows.add(new QualityResult(candidate.name(), candidate.resultLanguage(), mode,
|
||||
OutputPolicy.PRIMARY_OUTPUT, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0,
|
||||
new PartitionMetrics(1.0, 1.0, 1.0, 1.0, 1.0)));
|
||||
}
|
||||
@@ -77,10 +95,12 @@ final class QualityStemmerMatrixTest {
|
||||
final Path report = this.temporaryDirectory.resolve("matrix.csv");
|
||||
QualityReportWriter.writeCsv(report, rows);
|
||||
final String text = Files.readString(report, StandardCharsets.UTF_8);
|
||||
assertEquals(185, text.lines().count());
|
||||
assertEquals(197, text.lines().count());
|
||||
for (Candidate candidate : QualityStemmerMatrix.candidates()) {
|
||||
assertTrue(text.contains("\"" + candidate.name() + "\",\"" + candidate.language() + "\",\"ALL_WORDS\""));
|
||||
assertTrue(text.contains("\"" + candidate.name() + "\",\"" + candidate.language() + "\",\"LOWERCASE_GROUPS_ONLY\""));
|
||||
final String prefix = "\"" + candidate.name() + "\",\"" + candidate.resultLanguage()
|
||||
+ "\",\"\",\"\",\"\",";
|
||||
assertTrue(text.contains(prefix + "\"ALL_WORDS\""));
|
||||
assertTrue(text.contains(prefix + "\"LOWERCASE_GROUPS_ONLY\""));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ package org.egothor.stemmer.benchmark.quality;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumMap;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -44,6 +43,8 @@ import java.util.Set;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.egothor.stemmer.StemmerModelDescriptor;
|
||||
import org.egothor.stemmer.StemmerModelRegistry;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
import org.egothor.stemmer.benchmark.QualityStemmerMatrix;
|
||||
import org.egothor.stemmer.benchmark.QualityStemmerMatrix.Candidate;
|
||||
@@ -96,7 +97,7 @@ public final class StemmingQualityApplication {
|
||||
for (ProcessingMode mode : modes) {
|
||||
for (OutputPolicy policy : policies) {
|
||||
if (policy == OutputPolicy.PRIMARY_OUTPUT || multiple) {
|
||||
expected.add(new ResultKey(candidate.name(), candidate.language().name(), mode, policy));
|
||||
expected.add(new ResultKey(candidate.name(), candidate.resultLanguage(), mode, policy));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,15 +105,17 @@ public final class StemmingQualityApplication {
|
||||
|
||||
LOGGER.log(Level.INFO, filtered ? "Starting a filtered stemming-quality report."
|
||||
: "Starting the complete stemming-quality report.");
|
||||
final Map<Language, List<GoldStandardGroup>> dictionaries = new EnumMap<>(Language.class);
|
||||
final Map<String, List<GoldStandardGroup>> dictionaries = new HashMap<>();
|
||||
final List<QualityResult> results = new ArrayList<>();
|
||||
final List<QualityAudit.Scenario> audits = new ArrayList<>();
|
||||
final List<CandidateQualityAudit.Scenario> candidateAudits = new ArrayList<>();
|
||||
final StemmerModelRegistry modelRegistry = StemmerModelRegistry.fromContextClassLoader();
|
||||
for (Candidate candidate : candidates) {
|
||||
List<GoldStandardGroup> groups = dictionaries.get(candidate.language());
|
||||
final StemmerModelDescriptor model = modelRegistry.require(candidate.dictionaryModelId());
|
||||
List<GoldStandardGroup> groups = dictionaries.get(candidate.dictionaryModelId());
|
||||
if (groups == null) {
|
||||
groups = BundledGoldStandardLoader.load(candidate.language());
|
||||
dictionaries.put(candidate.language(), groups);
|
||||
groups = BundledGoldStandardLoader.loadModel(candidate.dictionaryModelId());
|
||||
dictionaries.put(candidate.dictionaryModelId(), groups);
|
||||
}
|
||||
for (ProcessingMode mode : modes) {
|
||||
final QualityStemmerMatrix.BatchStemmer primaryStemmer = candidate.createStemmer();
|
||||
@@ -120,27 +123,34 @@ public final class StemmingQualityApplication {
|
||||
if (audit && policies.contains(OutputPolicy.PRIMARY_OUTPUT)) {
|
||||
final QualityAudit.Scenario scenario = QualityAudit.evaluate(candidate, mode, groups, auditLimit);
|
||||
audits.add(scenario);
|
||||
primary = scenario.result();
|
||||
primary = withModelProvenance(scenario.result(), model);
|
||||
} else {
|
||||
primary = QualityEvaluator.evaluateBatch(candidate.name(), candidate.language().name(),
|
||||
mode, groups, primaryStemmer);
|
||||
primary = withModelProvenance(
|
||||
QualityEvaluator.evaluateBatch(candidate.name(), candidate.resultLanguage(),
|
||||
mode, groups, primaryStemmer),
|
||||
model);
|
||||
}
|
||||
if (policies.contains(OutputPolicy.PRIMARY_OUTPUT)) {
|
||||
results.add(primary);
|
||||
logScenario(candidate, mode, OutputPolicy.PRIMARY_OUTPUT);
|
||||
}
|
||||
if (multiOutput.get(candidate)) {
|
||||
final QualityResult anyCandidate = CandidateAwareEvaluator.evaluate(candidate.name(),
|
||||
candidate.language().name(), mode, OutputPolicy.ANY_CANDIDATE, groups, candidate.createStemmer());
|
||||
final QualityResult anyCandidate = withModelProvenance(
|
||||
CandidateAwareEvaluator.evaluate(candidate.name(),
|
||||
candidate.resultLanguage(), mode, OutputPolicy.ANY_CANDIDATE, groups,
|
||||
candidate.createStemmer()),
|
||||
model);
|
||||
final QualityResult allCandidates;
|
||||
if (audit) {
|
||||
final CandidateQualityAudit.Scenario scenario = CandidateQualityAudit.evaluate(
|
||||
candidate, mode, groups, primary, anyCandidate, auditLimit);
|
||||
candidateAudits.add(scenario);
|
||||
allCandidates = scenario.candidate();
|
||||
allCandidates = withModelProvenance(scenario.candidate(), model);
|
||||
} else {
|
||||
allCandidates = CandidateAwareEvaluator.evaluate(candidate.name(), candidate.language().name(),
|
||||
mode, OutputPolicy.ALL_CANDIDATES, groups, candidate.createStemmer());
|
||||
allCandidates = withModelProvenance(
|
||||
CandidateAwareEvaluator.evaluate(candidate.name(), candidate.resultLanguage(),
|
||||
mode, OutputPolicy.ALL_CANDIDATES, groups, candidate.createStemmer()),
|
||||
model);
|
||||
}
|
||||
verifyPolicyInvariants(primary, anyCandidate, allCandidates);
|
||||
if (policies.contains(OutputPolicy.ANY_CANDIDATE)) {
|
||||
@@ -175,15 +185,43 @@ public final class StemmingQualityApplication {
|
||||
LOGGER.log(Level.INFO, "Completed the stemming-quality report with {0} evaluated scenarios.", results.size());
|
||||
}
|
||||
|
||||
/** Selects candidates directly from the authoritative JMH matrix. */
|
||||
private static List<Candidate> selectCandidates(final Set<Language> languages, final String filter) {
|
||||
/** Attaches the exact independently versioned model used by one scenario. */
|
||||
private static QualityResult withModelProvenance(final QualityResult result,
|
||||
final StemmerModelDescriptor model) {
|
||||
return result.withModelProvenance(model.id(), model.version(), model.sha256());
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects candidates directly from the authoritative JMH matrix.
|
||||
*
|
||||
* <p>
|
||||
* An unfiltered publication run evaluates only each language's registered
|
||||
* default model. Optional model variants remain available only through an
|
||||
* explicit stemmer or model-ID filter and therefore cannot enter the complete
|
||||
* documentation snapshot accidentally.
|
||||
* </p>
|
||||
*/
|
||||
static List<Candidate> selectCandidates(final Set<Language> languages, final String filter) {
|
||||
return QualityStemmerMatrix.candidates().stream()
|
||||
.filter(candidate -> languages.contains(candidate.language()))
|
||||
.filter(candidate -> filter.isBlank() || candidate.name().equalsIgnoreCase(filter)
|
||||
|| candidate.name().toUpperCase(Locale.ROOT).endsWith("_" + filter.toUpperCase(Locale.ROOT)))
|
||||
.filter(candidate -> !filter.isBlank()
|
||||
|| candidate.dictionaryModelId().equals(candidate.language().defaultModelId()))
|
||||
.filter(candidate -> matchesFilter(candidate, filter))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/** Tests one candidate against the exact, suffix, and model-ID filters. */
|
||||
private static boolean matchesFilter(final Candidate candidate, final String filter) {
|
||||
if (filter.isBlank()) {
|
||||
return true;
|
||||
}
|
||||
final String normalizedFilter = filter.toUpperCase(Locale.ROOT);
|
||||
final String name = candidate.name().toUpperCase(Locale.ROOT);
|
||||
final String model = candidate.dictionaryModelId().toUpperCase(Locale.ROOT);
|
||||
return name.equals(normalizedFilter) || name.endsWith("_" + normalizedFilter)
|
||||
|| model.equals(normalizedFilter) || model.endsWith("-" + normalizedFilter);
|
||||
}
|
||||
|
||||
/** Parses a comma-separated language filter or selects every language. */
|
||||
private static Set<Language> parseLanguages(final String filter) {
|
||||
if (filter.isBlank()) {
|
||||
@@ -268,7 +306,7 @@ public final class StemmingQualityApplication {
|
||||
private static void logScenario(final Candidate candidate, final ProcessingMode mode, final OutputPolicy policy) {
|
||||
if (LOGGER.isLoggable(Level.INFO)) {
|
||||
LOGGER.log(Level.INFO, "Completed stemming-quality evaluation for stemmer {0}, language {1}, dictionary mode {2}, and output policy {3}.",
|
||||
new Object[] {candidate.name(), candidate.language(), mode, policy});
|
||||
new Object[] {candidate.name(), candidate.resultLanguage(), mode, policy});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,8 @@ import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
|
||||
/**
|
||||
* Publishes validated stemming-quality CSV results into marked sections of the
|
||||
* existing language benchmark pages. This test-source utility never modifies
|
||||
@@ -182,7 +184,9 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
throw new IllegalStateException("The stemming-quality CSV is empty.");
|
||||
}
|
||||
final List<String> header = parseCsv(lines.getFirst());
|
||||
final List<String> required = List.of("Stemmer", "Language", "Dictionary mode", "Output policy", "Applied dictionary rows",
|
||||
final List<String> required = List.of("Stemmer", "Language", "Dictionary model ID",
|
||||
"Dictionary model version", "Dictionary model SHA-256",
|
||||
"Dictionary mode", "Output policy", "Applied dictionary rows",
|
||||
"Processed word forms", "Forms with multiple candidates", "Maximum candidates for one form", "Total candidate assignments",
|
||||
"True-positive pairs", "False-positive pairs", "False-negative pairs", "True-negative pairs",
|
||||
"Over-stemming error pairs", "Over-stemming possible pairs", "Over-stemming percentage", "Under-stemming error pairs",
|
||||
@@ -244,6 +248,14 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
if (!keys.add(row.key())) {
|
||||
throw new IllegalStateException("Duplicate stemming-quality result key: " + row.key());
|
||||
}
|
||||
final String expectedModelId = Language.valueOf(row.language()).defaultModelId();
|
||||
if (!row.modelId().equals(expectedModelId)) {
|
||||
throw new IllegalStateException("Stemming-quality row " + row.key()
|
||||
+ " uses model " + row.modelId() + " instead of default model " + expectedModelId + ".");
|
||||
}
|
||||
if (row.modelVersion().isBlank() || !row.modelSha256().matches("[0-9a-f]{64}")) {
|
||||
throw new IllegalStateException("Incomplete dictionary-model provenance for " + row.key() + ".");
|
||||
}
|
||||
row.validate();
|
||||
}
|
||||
final Set<String> resultLanguages = new HashSet<>();
|
||||
@@ -275,8 +287,9 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
}
|
||||
validatePolicies(languageRows);
|
||||
}
|
||||
if (!documentedLanguages.contains("DA_DK") || !documentedLanguages.contains("YI")) {
|
||||
throw new IllegalStateException("The documentation mapping must contain DA_DK and YI.");
|
||||
if (!documentedLanguages.contains("DA_DK") || !documentedLanguages.contains("HE_IL")
|
||||
|| !documentedLanguages.contains("YI")) {
|
||||
throw new IllegalStateException("The documentation mapping must contain DA_DK, HE_IL, and YI.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,13 +317,15 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
|
||||
/** Renders one complete generated section for a language page. */
|
||||
private static String render(final Page page, final List<ResultRow> rows, final String checksum) {
|
||||
final String modelId = Language.valueOf(page.language()).defaultModelId();
|
||||
final StringBuilder output = new StringBuilder(32768);
|
||||
output.append(START).append("\n\n## Stemming Quality\n\n")
|
||||
.append("Runtime performance and linguistic grouping quality are independent dimensions. This section evaluates language `")
|
||||
.append(page.language()).append("` using the complete validated stemming-quality result matrix. Every usable dictionary row is one gold-standard group of forms expected to share a morphological family or lemma. Exact equality with a predetermined lemma is not required. Same-row pairs are positive pairs; pairs from different rows are negative pairs.\n\n")
|
||||
.append(page.language()).append("` using the complete validated stemming-quality result matrix. Every distinct surface form is one evaluated item and can belong to several dictionary groups. Two forms are a positive pair when their group-membership sets intersect and a negative pair when those sets are disjoint. A pair shared through several groups is counted once. Exact equality with a predetermined lemma is not required.\n\n")
|
||||
.append("`ALL_WORDS` includes every valid group and its original forms. `LOWERCASE_GROUPS_ONLY` excludes an entire group when any Unicode code point is uppercase or titlecase; retained words are not lowercased or otherwise rewritten. This isolates case-handling effects without changing retained inputs. [Download the complete machine-readable result snapshot](../data/stemming-quality.csv).\n\n")
|
||||
.append("### Evaluation Scope and Key Findings\n\n")
|
||||
.append("The dictionary resource is `src/main/resources/").append(page.language().toLowerCase(Locale.ROOT)).append("/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.\n\n");
|
||||
.append("The default model is `").append(modelId).append("`, loaded from classpath resource `org/egothor/stemmer/models/")
|
||||
.append(modelId).append("/stemmer.gz`. The following findings compare only deterministic `PRIMARY_OUTPUT` rows over identical included groups; candidate policies are reported separately as capability analyses.\n\n");
|
||||
for (String mode : MODES) {
|
||||
appendFinding(output, rows, mode);
|
||||
}
|
||||
@@ -320,13 +335,17 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
final long policies = selected.stream().map(ResultRow::policy).distinct().count();
|
||||
output.append("### `").append(mode).append("`\n\n")
|
||||
.append("This mode contains **").append(selected.size()).append(" result rows**, **").append(stemmers)
|
||||
.append(" evaluated stemmers**, and **").append(policies).append(" output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. Rankings are separated by output policy and ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. Balanced accuracy is a navigation metric, not a universally authoritative quality score.\n\n");
|
||||
.append(" evaluated stemmers**, and **").append(policies).append(" output policies**. Applied-row and form counts are shown per row because adapters share the language corpus but policy rows remain independently auditable. `PRIMARY_OUTPUT` and `ALL_CANDIDATES` rankings are ordered by unrounded balanced accuracy, followed by MCC, F1, over-stemming rate, over-stemming count, under-stemming rate, and stemmer. `ANY_CANDIDATE` has no single rank metric and is listed alphabetically. Balanced accuracy is a navigation metric, not a universally authoritative quality score.\n\n");
|
||||
for (String policy : List.of("PRIMARY_OUTPUT", "ANY_CANDIDATE", "ALL_CANDIDATES")) {
|
||||
final List<ResultRow> policyRows = selected.stream().filter(row -> row.policy().equals(policy)).toList();
|
||||
if (!policyRows.isEmpty()) {
|
||||
output.append("#### `").append(policy).append("` ranking\n\n");
|
||||
renderPrimaryTable(output, policyRows);
|
||||
renderDetailedTables(output, policyRows);
|
||||
if (policy.equals("ANY_CANDIDATE")) {
|
||||
renderAnyCandidatePolicy(output, policyRows);
|
||||
} else {
|
||||
output.append("#### `").append(policy).append("` ranking\n\n");
|
||||
renderPrimaryTable(output, policyRows);
|
||||
renderDetailedTables(output, policyRows);
|
||||
}
|
||||
}
|
||||
}
|
||||
renderCandidateAnalysis(output, selected);
|
||||
@@ -335,11 +354,12 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
output.append("### Provenance\n\n")
|
||||
.append("- Authoritative source: `docs/benchmarks/data/stemming-quality.csv`\n")
|
||||
.append("- Source SHA-256: `").append(checksum).append("`\n")
|
||||
.append("- Evaluation command: `./gradlew stemmingQuality`\n")
|
||||
.append("- Evaluation command: `./gradlew stemmingQuality --no-daemon`\n")
|
||||
.append("- Dictionary language: `").append(page.language()).append("`\n")
|
||||
.append("- Processing modes: `ALL_WORDS`, `LOWERCASE_GROUPS_ONLY`\n")
|
||||
.append("- Stemmer versions and transitive artifacts: resolved by the repository's JMH Gradle configuration and `gradle.lockfile`\n")
|
||||
.append("- Radixor version, Git revision, generation date, JDK version, operating system, and dictionary revision: not recorded in the authoritative CSV\n\n")
|
||||
.append("- Model ID, version, and SHA-256: recorded in every CSV row\n")
|
||||
.append("- Run date, core source state, JDK, operating system, and hardware: recorded on the [benchmark environment page](../reference/environment.md)\n\n")
|
||||
.append(END).append('\n');
|
||||
return output.toString();
|
||||
}
|
||||
@@ -366,24 +386,22 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
output.append(". This rank does not imply leadership in throughput or every secondary metric.\n");
|
||||
}
|
||||
|
||||
/** Renders the compact primary ranking table in an accessible scroll region. */
|
||||
/** Renders the compact primary ranking without duplicating metrics available in the details. */
|
||||
private static void renderPrimaryTable(final StringBuilder output, final List<ResultRow> rows) {
|
||||
output.append("<div class=\"quality-table quality-table--compact\" role=\"region\" aria-label=\"Compact stemming-quality ranking; scroll horizontally for additional columns\" tabindex=\"0\" markdown=\"1\">\n\n")
|
||||
.append("| Rank | Stemmer | Output policy | Balanced accuracy | Over-stemming | Under-stemming | F0.5 | F1 | MCC |\n")
|
||||
.append("|---:|---|---|---:|---:|---:|---:|---:|---:|\n");
|
||||
output.append("<div class=\"quality-summary\" markdown=\"1\">\n\n")
|
||||
.append("| Rank | Stemmer | Balanced accuracy | Over-stemming (OI) | Under-stemming (UI) |\n")
|
||||
.append("|---:|---|---:|---:|---:|\n");
|
||||
for (int index = 0; index < rows.size(); index++) {
|
||||
final ResultRow row = rows.get(index);
|
||||
output.append('|').append(index + 1).append('|').append(displayStemmer(row.stemmer())).append('|').append(row.policy()).append('|')
|
||||
output.append('|').append(index + 1).append('|').append(displayStemmer(row.stemmer())).append('|')
|
||||
.append(metric(row, "Balanced accuracy")).append('|')
|
||||
.append(pair(row, "Over-stemming error pairs", "Over-stemming possible pairs", "Over-stemming percentage")).append('|')
|
||||
.append(pair(row, "Under-stemming error pairs", "Under-stemming possible pairs", "Under-stemming percentage")).append('|')
|
||||
.append(metric(row, "Pairwise F0.5")).append('|').append(metric(row, "Pairwise F1")).append('|')
|
||||
.append(metric(row, "Matthews correlation coefficient")).append("|\n");
|
||||
.append(rate(row, "Over-stemming error pairs", "Over-stemming percentage")).append('|')
|
||||
.append(rate(row, "Under-stemming error pairs", "Under-stemming percentage")).append("|\n");
|
||||
}
|
||||
output.append("\n</div>\n\n");
|
||||
}
|
||||
|
||||
/** Renders classification, relation, partition, and raw-count tables with repeated identities. */
|
||||
/** Renders classification, relation, and raw-count tables with repeated identities. */
|
||||
private static void renderDetailedTables(final StringBuilder output, final List<ResultRow> rows) {
|
||||
output.append("<details class=\"quality-details\" markdown=\"1\"><summary>Classification metrics</summary>\n\n")
|
||||
.append("| Rank | Stemmer | Output policy | Precision | Recall | Specificity | Balanced accuracy | Pairwise accuracy | Error rate |\n")
|
||||
@@ -403,22 +421,40 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
.append(metric(row, "Pairwise F2")).append('|').append(metric(row, "Jaccard index")).append('|')
|
||||
.append(metric(row, "Fowlkes-Mallows index")).append('|').append(metric(row, "Matthews correlation coefficient")).append("|\n");
|
||||
}
|
||||
output.append("\n</details>\n\n<details class=\"quality-details\" markdown=\"1\"><summary>Partition metrics (PRIMARY_OUTPUT only)</summary>\n\n")
|
||||
.append("| Rank | Stemmer | Output policy | Adjusted Rand Index | Homogeneity | Completeness | V-measure | Normalized mutual information |\n")
|
||||
.append("|---:|---|---|---:|---:|---:|---:|---:|\n");
|
||||
for (int index = 0; index < rows.size(); index++) {
|
||||
final ResultRow row = rows.get(index);
|
||||
output.append(identity(index, row)).append(metric(row, "Adjusted Rand Index")).append('|').append(metric(row, "Homogeneity")).append('|')
|
||||
.append(metric(row, "Completeness")).append('|').append(metric(row, "V-measure")).append('|')
|
||||
.append(metric(row, "Normalized mutual information")).append("|\n");
|
||||
}
|
||||
output.append("\n</details>\n\n<details class=\"quality-details\" markdown=\"1\"><summary>Raw pair counts</summary>\n\n")
|
||||
.append("| Rank | Stemmer | Output policy | TP | FP | FN | TN | Over error / possible | Under error / possible |\n")
|
||||
.append("|---:|---|---|---:|---:|---:|---:|---:|---:|\n");
|
||||
for (int index = 0; index < rows.size(); index++) {
|
||||
final ResultRow row = rows.get(index);
|
||||
output.append(identity(index, row)).append(row.value("True-positive pairs")).append('|').append(row.value("False-positive pairs"))
|
||||
.append('|').append(row.value("False-negative pairs")).append('|').append(row.value("True-negative pairs")).append('|')
|
||||
output.append(identity(index, row)).append(rawCount(row, "True-positive pairs")).append('|')
|
||||
.append(rawCount(row, "False-positive pairs")).append('|')
|
||||
.append(rawCount(row, "False-negative pairs")).append('|')
|
||||
.append(rawCount(row, "True-negative pairs")).append('|')
|
||||
.append(row.value("Over-stemming error pairs")).append(" / ").append(row.value("Over-stemming possible pairs")).append('|')
|
||||
.append(row.value("Under-stemming error pairs")).append(" / ").append(row.value("Under-stemming possible pairs")).append("|\n");
|
||||
}
|
||||
output.append("\n</details>\n\n");
|
||||
}
|
||||
|
||||
/** Renders the two defined per-pair oracle bounds without implying one confusion matrix. */
|
||||
private static void renderAnyCandidatePolicy(final StringBuilder output, final List<ResultRow> rows) {
|
||||
final List<ResultRow> alphabetical = rows.stream().sorted(Comparator.comparing(ResultRow::stemmer)).toList();
|
||||
output.append("#### `ANY_CANDIDATE` oracle bounds\n\n")
|
||||
.append("These results are measured, not missing. `ANY_CANDIDATE` answers two separate optimistic questions for each pair: a gold-related pair avoids under-stemming when the candidate sets intersect, while a gold-negative pair avoids over-stemming when some non-colliding candidate selection exists. The oracle may choose a different candidate for the same word in different pairs. Consequently, these decisions do not form one globally realizable predicted relation or one TP/FP/FN/TN confusion matrix. Balanced accuracy, F-scores, Jaccard, Fowlkes–Mallows, and MCC are therefore mathematically **not applicable**, rather than unknown.\n\n")
|
||||
.append("<div class=\"quality-summary quality-summary--oracle\" markdown=\"1\">\n\n")
|
||||
.append("| Stemmer | Optimistic over-stemming (OI) | Optimistic under-stemming (UI) |\n")
|
||||
.append("|---|---:|---:|\n");
|
||||
for (ResultRow row : alphabetical) {
|
||||
output.append('|').append(displayStemmer(row.stemmer())).append('|')
|
||||
.append(rate(row, "Over-stemming error pairs", "Over-stemming percentage")).append('|')
|
||||
.append(rate(row, "Under-stemming error pairs", "Under-stemming percentage")).append("|\n");
|
||||
}
|
||||
output.append("\n</div>\n\n")
|
||||
.append("<details class=\"quality-details\" markdown=\"1\"><summary>Oracle-bound pair counts</summary>\n\n")
|
||||
.append("| Stemmer | Unavoidable over errors / gold-negative pairs | Unrepairable under errors / gold-related pairs |\n")
|
||||
.append("|---|---:|---:|\n");
|
||||
for (ResultRow row : alphabetical) {
|
||||
output.append('|').append(displayStemmer(row.stemmer())).append('|')
|
||||
.append(row.value("Over-stemming error pairs")).append(" / ").append(row.value("Over-stemming possible pairs")).append('|')
|
||||
.append(row.value("Under-stemming error pairs")).append(" / ").append(row.value("Under-stemming possible pairs")).append("|\n");
|
||||
}
|
||||
@@ -464,13 +500,13 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
/** Appends the self-contained policy, confusion-matrix, and metric definitions. */
|
||||
private static void appendMethodology(final StringBuilder output) {
|
||||
output.append("### Output Policies and Metric Definitions\n\n")
|
||||
.append("`PRIMARY_OUTPUT` uses one deterministic stem per form and therefore defines a strict partition. `ANY_CANDIDATE` is an optimistic oracle-assisted pairwise upper bound: a same-group pair succeeds when candidates intersect, while a different-group pair succeeds when a non-colliding selection exists. Candidate choices may differ between pairs, so this is not deterministic runtime behaviour and need not represent one globally consistent assignment. `ALL_CANDIDATES` activates every returned candidate; forms are related when candidate sets intersect. Alternatives can reduce under-stemming but can introduce cross-group collisions, and the resulting relation can overlap and need not be a partition.\n\n")
|
||||
.append("For each row, `TP = underPossiblePairs - underErrorPairs`, `FN = underErrorPairs`, `FP = overErrorPairs`, and `TN = overPossiblePairs - overErrorPairs`. TP and FN concern same-group pairs; FP and TN concern different-group pairs. Consequently, under-stemming and over-stemming use different denominators. Undefined values are rendered as `n/a`.\n\n")
|
||||
.append("- Under-stemming rate: `FN / (TP + FN)`, the false-negative rate over same-group pairs.\n")
|
||||
.append("- Over-stemming rate: `FP / (TN + FP)`, the false-positive rate over different-group pairs.\n")
|
||||
.append("Each distinct surface form is one item and may belong to several gold groups. Two forms are gold-related when their membership sets intersect; a relation shared by several groups is counted once. `PRIMARY_OUTPUT` uses one deterministic stem per form. `ANY_CANDIDATE` is an optimistic oracle-assisted pairwise upper bound: a gold-related pair succeeds when candidates intersect, while a gold-negative pair succeeds when a non-colliding selection exists. Candidate choices may differ between pairs, so this is not deterministic runtime behaviour and does not define one confusion matrix. `ALL_CANDIDATES` activates every returned candidate; forms are related when candidate sets intersect.\n\n")
|
||||
.append("For `PRIMARY_OUTPUT` and `ALL_CANDIDATES`, `TP = underPossiblePairs - underErrorPairs`, `FN = underErrorPairs`, `FP = overErrorPairs`, and `TN = overPossiblePairs - overErrorPairs`. `ANY_CANDIDATE` publishes only its separate oracle-assisted under/over bounds; confusion-derived metrics are mathematically inapplicable and are not presented in its language-page section. Their machine-readable CSV fields remain empty. Undefined metric denominators in otherwise applicable policies are rendered as `n/a`.\n\n")
|
||||
.append("- Under-stemming rate (Paice UI): `FN / (TP + FN)`, the false-negative rate over gold-related pairs.\n")
|
||||
.append("- Over-stemming rate (Paice OI): `FP / (TN + FP)`, the false-positive rate over gold-negative pairs.\n")
|
||||
.append("- Pairwise precision: `TP / (TP + FP)`, the fraction of predicted conflations that are gold-standard positive pairs.\n")
|
||||
.append("- Pairwise recall: `TP / (TP + FN)`, the fraction of gold-standard positive pairs successfully connected.\n")
|
||||
.append("- Pairwise specificity: `TN / (TN + FP)`, the fraction of different-group pairs correctly separated.\n")
|
||||
.append("- Pairwise specificity: `TN / (TN + FP)`, the fraction of gold-negative pairs correctly separated.\n")
|
||||
.append("- Balanced accuracy: `(recall + specificity) / 2`. It gives equal weight to positive and negative pair classes and is less dominated by the large true-negative class than ordinary accuracy. It does not replace the raw errors or other metrics.\n")
|
||||
.append("- Pairwise F-beta: `((1 + betaSquared) * TP) / (((1 + betaSquared) * TP) + (betaSquared * FN) + FP)`. F0.5 emphasizes precision and penalizes over-stemming more; F1 weights precision and recall equally; F2 emphasizes recall and penalizes under-stemming more.\n")
|
||||
.append("- MCC: `(TP * TN - FP * FN) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))`. It uses all confusion counts and remains useful under class imbalance, except when its denominator is degenerate.\n")
|
||||
@@ -478,14 +514,15 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
.append("- Fowlkes–Mallows index: `sqrt(precision * recall)`.\n")
|
||||
.append("- Pairwise accuracy: `(TP + TN) / (TP + TN + FP + FN)`. It can be dominated by true-negative cross-group pairs.\n")
|
||||
.append("- Pairwise error rate: `(FP + FN) / (TP + TN + FP + FN)`.\n\n")
|
||||
.append("Adjusted Rand Index uses the gold/predicted contingency table and chance correction. Homogeneity is `1 - H(gold | predicted) / H(gold)`; completeness is `1 - H(predicted | gold) / H(predicted)`; V-measure is their harmonic mean; normalized mutual information uses the arithmetic-mean entropy normalization `MI / ((H(gold) + H(predicted)) / 2)`. These partition-only metrics apply to `PRIMARY_OUTPUT`; candidate-relation rows show `n/a`.\n\n");
|
||||
.append("Standard ARI, homogeneity, completeness, V-measure, and NMI are not calculated: their usual contingency-table definitions require an exclusive gold partition, while this gold standard is an overlapping cover.\n\n");
|
||||
}
|
||||
|
||||
/** Renders the generated executive findings, winner matrix, and Radixor aggregates. */
|
||||
private static String renderOverview(final Map<String, Page> pages, final List<ResultRow> rows, final String checksum) {
|
||||
final StringBuilder output = new StringBuilder(16384);
|
||||
output.append(OVERVIEW_START).append("\n\n## Pairwise Quality Findings\n\n")
|
||||
.append("The validated snapshot is a broad multilingual comparison covering the complete 20-language Radixor dictionary universe; 19 languages have existing benchmark pages. The direct ranking below uses only deterministic `PRIMARY_OUTPUT` rows over identical per-language inputs. Candidate-aware rows are intentionally excluded from this claim.\n\n");
|
||||
.append("The validated snapshot is a broad multilingual comparison covering the complete ")
|
||||
.append(pages.size()).append("-language Radixor default-model universe, with one benchmark page per language. The direct ranking below uses only deterministic `PRIMARY_OUTPUT` rows over identical per-language inputs. Candidate-aware rows are intentionally excluded from this claim.\n\n");
|
||||
int radixorWins = 0;
|
||||
int comparisons = 0;
|
||||
for (String mode : MODES) {
|
||||
@@ -524,7 +561,8 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
for (String mode : MODES) {
|
||||
renderPlacementSummary(output, pages, rows, mode);
|
||||
}
|
||||
output.append("\n### Radixor full-coverage aggregates\n\nThese aggregates cover all 19 documented languages. Macro balanced accuracy gives each language equal weight. Micro metrics first sum raw pair counts across languages. Unsupported third-party languages are never inserted as zero results, so this full-coverage table is not presented as a cross-stemmer common-language ranking.\n\n")
|
||||
output.append("\n### Radixor full-coverage aggregates\n\nThese aggregates cover all ")
|
||||
.append(pages.size()).append(" documented languages. Macro balanced accuracy gives each language equal weight. Micro metrics first sum raw pair counts across languages. Unsupported third-party languages are never inserted as zero results, so this full-coverage table is not presented as a cross-stemmer common-language ranking.\n\n")
|
||||
.append("| Dictionary mode | Languages | Macro balanced accuracy | Micro balanced accuracy | Micro precision | Micro recall | Micro F1 |\n")
|
||||
.append("|---|---:|---:|---:|---:|---:|---:|\n");
|
||||
for (String mode : MODES) {
|
||||
@@ -669,12 +707,28 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
return value.isEmpty() ? "n/a" : String.format(Locale.ROOT, "%.6f", Double.parseDouble(value));
|
||||
}
|
||||
|
||||
/** Formats an over- or under-stemming rate as a percentage. */
|
||||
private static String rate(final ResultRow row, final String errorName, final String percentageName) {
|
||||
final String value = row.value(percentageName);
|
||||
if (value.isEmpty()) {
|
||||
return "n/a";
|
||||
}
|
||||
final double rate = Double.parseDouble(value);
|
||||
return rate < 0.000001 && row.longValue(errorName) > 0 ? "<0.000001%" : String.format(Locale.ROOT, "%.6f%%", rate);
|
||||
}
|
||||
|
||||
/** Formats one raw error numerator, denominator, and percentage. */
|
||||
private static String pair(final ResultRow row, final String error, final String possible, final String percentage) {
|
||||
final String rate = row.value(percentage);
|
||||
return row.value(error) + " / " + row.value(possible) + " (" + (rate.isEmpty() ? "n/a" : String.format(Locale.ROOT, "%.6f%%", Double.parseDouble(rate))) + ")";
|
||||
}
|
||||
|
||||
/** Formats an inapplicable confusion count explicitly. */
|
||||
private static String rawCount(final ResultRow row, final String name) {
|
||||
final String value = row.value(name);
|
||||
return value.isEmpty() ? "n/a" : value;
|
||||
}
|
||||
|
||||
/** Replaces an existing marked section or appends the first generated section. */
|
||||
private static String replaceSection(final String original, final String section) {
|
||||
return replaceMarkedSection(original, section, START, END);
|
||||
@@ -725,6 +779,12 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
private String stemmer() { return value("Stemmer"); }
|
||||
/** Returns the language identifier. */
|
||||
private String language() { return value("Language"); }
|
||||
/** Returns the dictionary model identifier. */
|
||||
private String modelId() { return value("Dictionary model ID"); }
|
||||
/** Returns the dictionary model version. */
|
||||
private String modelVersion() { return value("Dictionary model version"); }
|
||||
/** Returns the dictionary model SHA-256. */
|
||||
private String modelSha256() { return value("Dictionary model SHA-256"); }
|
||||
/** Returns the dictionary-processing mode. */
|
||||
private String mode() { return value("Dictionary mode"); }
|
||||
/** Returns the output policy. */
|
||||
@@ -736,21 +796,34 @@ public final class StemmingQualityDocumentationPublisher {
|
||||
/** Parses a numeric field, placing undefined values last during sorting. */
|
||||
private double number(final String name) { return value(name).isEmpty() ? Double.NEGATIVE_INFINITY : Double.parseDouble(value(name)); }
|
||||
/** Returns false-negative pairs. */
|
||||
private long fn() { return longValue("False-negative pairs"); }
|
||||
private long fn() { return longValue("Under-stemming error pairs"); }
|
||||
/** Returns false-positive pairs. */
|
||||
private long fp() { return longValue("False-positive pairs"); }
|
||||
private long fp() { return longValue("Over-stemming error pairs"); }
|
||||
|
||||
/** Validates raw confusion counts and the published balanced accuracy. */
|
||||
private void validate() {
|
||||
final long tp = longValue("True-positive pairs");
|
||||
final long fp = fp();
|
||||
final long fn = fn();
|
||||
final long tn = longValue("True-negative pairs");
|
||||
if (fn != longValue("Under-stemming error pairs") || fp != longValue("Over-stemming error pairs")
|
||||
|| Math.addExact(tp, fn) != longValue("Under-stemming possible pairs")
|
||||
|| Math.addExact(tn, fp) != longValue("Over-stemming possible pairs")) {
|
||||
final long underPossible = longValue("Under-stemming possible pairs");
|
||||
final long overPossible = longValue("Over-stemming possible pairs");
|
||||
if (fn < 0 || fp < 0 || fn > underPossible || fp > overPossible) {
|
||||
throw new IllegalStateException("Raw pair-count invariants fail for " + key());
|
||||
}
|
||||
if (policy().equals("ANY_CANDIDATE")) {
|
||||
if (!value("True-positive pairs").isEmpty() || !value("False-positive pairs").isEmpty()
|
||||
|| !value("False-negative pairs").isEmpty() || !value("True-negative pairs").isEmpty()
|
||||
|| !value("Balanced accuracy").isEmpty() || !value("Pairwise F1").isEmpty()
|
||||
|| !value("Matthews correlation coefficient").isEmpty()) {
|
||||
throw new IllegalStateException("Oracle-assisted ANY_CANDIDATE row contains incoherent classification metrics: "
|
||||
+ key());
|
||||
}
|
||||
return;
|
||||
}
|
||||
final long tp = longValue("True-positive pairs");
|
||||
final long tn = longValue("True-negative pairs");
|
||||
if (Math.addExact(tp, fn) != underPossible || Math.addExact(tn, fp) != overPossible) {
|
||||
throw new IllegalStateException("Raw confusion-count invariants fail for " + key());
|
||||
}
|
||||
final double recall = ratio(tp, Math.addExact(tp, fn));
|
||||
final double specificity = ratio(tn, Math.addExact(tn, fp));
|
||||
final double expected = (recall + specificity) / 2.0;
|
||||
|
||||
Reference in New Issue
Block a user