feat(benchmarks): expand multilingual stemming quality evaluation

* cover all Radixor dictionary languages
* add PRIMARY_OUTPUT, ANY_CANDIDATE, and ALL_CANDIDATES policies
* measure pairwise over-stemming and under-stemming
* add balanced accuracy and complementary quality metrics
* compare single-output and multi-output stemmers fairly
* improve result validation, reporting, and documentation
* move stemming quality tests into the standard test source set
* preserve the existing JMH benchmark structure and badge output
This commit is contained in:
2026-07-20 23:20:17 +02:00
parent 6d35f01303
commit 05f3855b99
62 changed files with 11472 additions and 75 deletions

View File

@@ -30,6 +30,11 @@ apply from: 'gradle/maven-pom.gradle'
configurations {
mockitoAgent
stemmingQualityJmhRuntime {
canBeConsumed = false
canBeResolved = true
extendsFrom(jmhImplementation, jmhRuntimeOnly)
}
}
java {
@@ -78,6 +83,16 @@ dependencies {
}
}
sourceSets.jmh.compileClasspath = sourceSets.jmh.compileClasspath - sourceSets.test.output
sourceSets.jmh.runtimeClasspath = sourceSets.jmh.runtimeClasspath - sourceSets.test.output
sourceSets.test.compileClasspath += sourceSets.jmh.output + configurations.jmhCompileClasspath
sourceSets.test.runtimeClasspath += sourceSets.jmh.output + configurations.jmhCompileClasspath
tasks.named('compileJmhJava', JavaCompile) {
classpath = classpath - sourceSets.test.output
setDependsOn([tasks.named('classes')])
}
dependencyCheck {
failBuildOnCVSS = 7.0
failOnError = true
@@ -426,6 +441,7 @@ tasks.named('distTar') {
jmh {
jmhVersion = '1.37'
includeTests = false
warmupIterations = 3
iterations = 5
fork = 1
@@ -468,6 +484,85 @@ tasks.register('regressionArtifactGenerator', JavaExec) {
}
}
tasks.register('stemmingQuality', JavaExec) {
group = 'verification'
description = 'Evaluates pairwise over-stemming and under-stemming against bundled dictionary groups.'
dependsOn(tasks.named('testClasses'))
dependsOn(tasks.named('jmhClasses'))
classpath = files(sourceSets.test.runtimeClasspath, configurations.stemmingQualityJmhRuntime)
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityApplication'
args layout.buildDirectory.dir('reports/stemming-quality').get().asFile.absolutePath,
layout.projectDirectory.dir('src/main/resources').asFile.absolutePath,
providers.gradleProperty('stemmingQualityLanguage').getOrElse(''),
providers.gradleProperty('stemmingQualityStemmer').getOrElse(''),
providers.gradleProperty('stemmingQualityMode').getOrElse(''),
providers.gradleProperty('stemmingQualityOutputPolicy').getOrElse(''),
providers.gradleProperty('stemmingQualityRankMetric').getOrElse('PAIRWISE_F05'),
providers.gradleProperty('stemmingQualityAudit').getOrElse('false'),
providers.gradleProperty('stemmingQualityAuditLimit').getOrElse('25')
maxHeapSize = '6g'
}
tasks.register('publishStemmingQualityDocumentation', JavaExec) {
group = 'documentation'
description = 'Publishes validated complete stemming-quality results on the language benchmark pages.'
dependsOn(tasks.named('testClasses'))
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityDocumentationPublisher'
args layout.buildDirectory.file('reports/stemming-quality/stemming-quality.csv').get().asFile.absolutePath,
layout.projectDirectory.dir('docs').asFile.absolutePath,
'update'
doFirst {
if (!file("$buildDir/reports/stemming-quality/stemming-quality.csv").isFile()) {
throw new GradleException('A complete stemming-quality CSV is required. Run stemmingQuality only when no validated complete report is available.')
}
}
}
tasks.register('verifyStemmingQualityDocumentation', JavaExec) {
group = 'verification'
description = 'Verifies published language-page quality tables against the checked-in authoritative CSV.'
dependsOn(tasks.named('testClasses'))
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityDocumentationPublisher'
args layout.projectDirectory.file('docs/benchmarks/data/stemming-quality.csv').asFile.absolutePath,
layout.projectDirectory.dir('docs').asFile.absolutePath,
'verify'
}
tasks.named('check') {
dependsOn(tasks.named('verifyStemmingQualityDocumentation'))
}
tasks.register('verifyStemmingQualitySourceSets') {
group = 'verification'
description = 'Verifies the production, JMH, and standard-test ownership of stemming-quality infrastructure.'
doLast {
if (sourceSets.findByName('stemmingQualityTest') != null || file('src/stemmingQualityTest').exists()) {
throw new GradleException('The obsolete stemmingQualityTest source set or directory still exists.')
}
if (!file('src/jmh/java/org/egothor/stemmer/benchmark/QualityStemmerMatrix.java').isFile()) {
throw new GradleException('The authoritative JMH stemmer matrix is not in src/jmh.')
}
if (!file('src/test/java/org/egothor/stemmer/benchmark/quality/StemmingQualityApplication.java').isFile()) {
throw new GradleException('The stemming-quality evaluator is not in the standard test source set.')
}
}
}
tasks.register('verifyProductionJarExcludesStemmingQuality') {
group = 'verification'
description = 'Verifies that analytical stemming-quality classes are absent from the production JAR.'
dependsOn(tasks.named('jar'))
doLast {
final File archive = tasks.named('jar').get().archiveFile.get().asFile
final def forbidden = zipTree(archive).matching { include '**/benchmark/**' }.files
if (!forbidden.isEmpty()) {
throw new GradleException("Production JAR contains analytical stemming-quality classes: ${forbidden}")
}
}
}
tasks.register('printDependencyCheckNvdConfig') {
doLast {
System.out.println("NVD API key present: " + (nvdApiKey != null && !nvdApiKey.isBlank()))