Files
Radixor/build.gradle
Leo Galambos 5e3d3c7c7d feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility
- distribute standard compiled models as a separate Python package
- generate model artifacts during builds instead of storing them in Git
- add GitHub release and Pages-backed package index workflows
- add Python tests, benchmarks, documentation, and Gradle integration
- refresh the documentation site, branding, and language benchmarks
2026-08-10 22:34:32 +02:00

1160 lines
47 KiB
Groovy

plugins {
id 'org.egothor.radixor.build-support'
id 'java'
id 'eclipse'
id 'application'
id 'maven-publish'
id 'signing'
id 'pmd'
id 'jacoco'
id 'info.solidsoft.pitest' version '1.19.0'
id 'me.champeau.jmh' version '0.7.3'
id 'org.owasp.dependencycheck' version '12.2.1'
id 'org.cyclonedx.bom' version '3.3.0'
id 'com.palantir.git-version' version '4.0.0'
}
group = 'org.egothor'
version = gitVersion(prefix:'release@')
def benchmarkReportsDirectory = layout.buildDirectory.dir('reports/jmh')
def sbomReportsDirectory = layout.buildDirectory.dir('reports/sbom')
def jmhIncludesProperty = providers.gradleProperty('jmh.includes')
.orElse(providers.systemProperty('jmh.includes'))
def nvdApiKey = providers.gradleProperty('nvdApiKey')
.orElse(providers.environmentVariable('NVD_API_KEY'))
.orNull
def dependencyCheckSuppressionFile = rootProject.file('dependency-suppression.xml')
apply from: 'gradle/maven-pom.gradle'
configurations {
mockitoAgent
stemmingQualityJmhRuntime {
canBeConsumed = false
canBeResolved = true
extendsFrom(jmhImplementation, jmhRuntimeOnly)
}
}
java {
withSourcesJar()
withJavadocJar()
sourceCompatibility = JavaVersion.VERSION_21
targetCompatibility = JavaVersion.VERSION_21
}
tasks.withType(JavaCompile).configureEach {
options.compilerArgs.addAll(['-Xlint:deprecation', '-Xlint:unchecked'])
}
tasks.withType(AbstractArchiveTask).configureEach {
preserveFileTimestamps = false
reproducibleFileOrder = true
}
jacoco {
toolVersion = '0.8.14'
}
pmd {
consoleOutput = true
toolVersion = '7.20.0'
sourceSets = [sourceSets.main]
ruleSetFiles = files(rootProject.file(".ruleset"))
}
dependencyLocking {
lockAllConfigurations()
lockMode = LockMode.STRICT
}
dependencies {
jmhImplementation sourceSets.main.output
modelProjects().each { Project modelProject ->
testRuntimeOnly project(modelProject.path)
jmhRuntimeOnly project(modelProject.path)
}
testImplementation platform(libs.junit.bom)
testImplementation libs.junit.jupiter
testRuntimeOnly libs.junit.platform.launcher
testImplementation libs.mockito.core
testImplementation libs.mockito.junit.jupiter
testImplementation libs.jqwik
testImplementation gradleTestKit()
mockitoAgent(libs.mockito.core) {
transitive = false
}
}
def modelProjects() {
Properties topology = new Properties()
rootProject.file('models/model-projects.properties').withInputStream { InputStream input ->
topology.load(input)
}
return topology.stringPropertyNames().toList().sort().collect { String modelId ->
project(":models:${modelId}")
}
}
def defaultModelProjects() {
Properties topology = new Properties()
rootProject.file('models/model-projects.properties').withInputStream { InputStream input ->
topology.load(input)
}
return topology.stringPropertyNames().findAll { String modelId ->
topology.getProperty(modelId) == 'default'
}.sort().collect { String modelId -> project(":models:${modelId}") }
}
tasks.named('projects') {
actions.clear()
doLast {
logger.lifecycle('Root project \'{}\'', rootProject.name)
rootProject.allprojects.findAll { Project candidate -> candidate != rootProject }
.sort { Project left, Project right -> left.path <=> right.path }
.each { Project candidate -> logger.lifecycle('+--- Project \'{}\'', candidate.path) }
gradle.includedBuilds.toList().sort { left, right -> left.name <=> right.name }
.each { includedBuild -> logger.lifecycle('Included build \'{}\'', includedBuild.name) }
}
}
sourceSets.jmh.compileClasspath = sourceSets.jmh.compileClasspath - sourceSets.test.output
sourceSets.jmh.runtimeClasspath = sourceSets.jmh.runtimeClasspath - sourceSets.test.output
sourceSets.test.compileClasspath += sourceSets.jmh.output + configurations.jmhCompileClasspath
sourceSets.test.runtimeClasspath += sourceSets.jmh.output + configurations.jmhCompileClasspath
tasks.named('compileJmhJava', JavaCompile) {
classpath = classpath - sourceSets.test.output
setDependsOn([tasks.named('classes')])
}
dependencyCheck {
failBuildOnCVSS = 7.0
failOnError = true
autoUpdate = true
formats = ['HTML', 'JSON']
outputDirectory = layout.buildDirectory.dir('reports/dependency-check').get().asFile.absolutePath
/*
* Keep the scan focused on actual Java dependency inputs used by this project.
* testRuntimeClasspath is included intentionally because the current external
* dependency surface is primarily test-scoped.
*/
scanConfigurations = ['runtimeClasspath', 'testRuntimeClasspath', 'mockitoAgent']
skipTestGroups = false
analyzers {
experimentalEnabled = false
centralEnabled = true
}
nvd {
apiKey = nvdApiKey
delay = nvdApiKey != null ? 3500 : 8000
validForHours = 4
}
if (dependencyCheckSuppressionFile.exists()) {
suppressionFile = dependencyCheckSuppressionFile.absolutePath
failBuildOnUnusedSuppressionRule = true
}
}
def cliIncludeTags = project.findProperty('includeTags')?.toString() ?: System.getProperty('includeTags')
def cliExcludeTags = project.findProperty('excludeTags')?.toString() ?: System.getProperty('excludeTags')
def splitTagExpression = { String tagsExpr ->
if (tagsExpr == null || tagsExpr.isBlank()) {
return []
}
return tagsExpr.split(',')
.collect { it.trim() }
.findAll { it != null && !it.isBlank() }
}
tasks.withType(Test).configureEach {
final def mockitoAgentArguments = objects.newInstance(
org.egothor.radixor.MockitoAgentArgumentProvider)
mockitoAgentArguments.agentClasspath.from(configurations.mockitoAgent)
jvmArgumentProviders.add(mockitoAgentArguments)
/*
* Bundled dictionary integration tests compile and reload large real-world
* stemming dictionaries, including large language resources such as es_es.
* The default Gradle test executor heap is too small for this workload.
*/
minHeapSize = '1g'
maxHeapSize = '4g'
reports {
junitXml.required = true
html.required = true
}
}
tasks.named('test', Test) {
dependsOn('prepareModelConsumerTestRepository')
systemProperty('radixor.consumer.repository',
layout.buildDirectory.dir('model-consumer-repository').get().asFile.absolutePath)
systemProperty('radixor.core.version', version.toString())
systemProperty('radixor.catalog.version', project(':models:standard').version.toString())
}
tasks.register('modelDependencyResolutionTest', Test) {
group = 'verification'
description = 'Verifies that published model coordinates resolve from the generated consumer repository.'
dependsOn(tasks.named('prepareModelConsumerTestRepository'))
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
useJUnitPlatform()
filter {
includeTestsMatching('org.egothor.stemmer.ModelDependencyResolutionTest')
}
systemProperty('radixor.consumer.repository',
layout.buildDirectory.dir('model-consumer-repository').get().asFile.absolutePath)
systemProperty('radixor.core.version', version.toString())
systemProperty('radixor.catalog.version', project(':models:standard').version.toString())
}
def configureJUnitPlatformTags = { Test task, String includeTagsExpr, String excludeTagsExpr ->
task.useJUnitPlatform {
final def includes = splitTagExpression(includeTagsExpr)
final def excludes = splitTagExpression(excludeTagsExpr)
if (!includes.isEmpty()) {
includeTags(*includes.toArray(new String[0]))
}
if (!excludes.isEmpty()) {
excludeTags(*excludes.toArray(new String[0]))
}
}
}
tasks.named('test', Test) {
final def requestedIncludes = splitTagExpression(cliIncludeTags)
final boolean slowExplicitlyIncluded = requestedIncludes.contains('slow')
final String defaultExcludeTags = cliExcludeTags ?: (slowExplicitlyIncluded ? 'large-model' : 'slow,large-model')
configureJUnitPlatformTags(it, cliIncludeTags, defaultExcludeTags)
finalizedBy(tasks.named('jacocoTestReport'))
}
def largeModelMaxHeap = providers.gradleProperty('radixorLargeModelMaxHeap').orElse('6g')
def runtimeModelId = providers.gradleProperty('modelId').orElse('pl-pl-polimorf')
def runtimeModelClasspath = configurations.testRuntimeClasspath.incoming.artifactView {
componentFilter { componentIdentifier ->
if (!(componentIdentifier instanceof org.gradle.api.artifacts.component.ProjectComponentIdentifier)) {
return true
}
final String projectPath = componentIdentifier.projectPath
return !projectPath.startsWith(':models:') || projectPath == ":models:${runtimeModelId.get()}"
}
}.files
tasks.register('runtimeModelIntegrationTest', Test) {
group = 'verification'
description = 'Constructs one complete selected runtime model in an isolated, memory-sized JVM.'
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.output + sourceSets.main.output + sourceSets.jmh.output + runtimeModelClasspath
dependsOn(tasks.named('compileTestJava'))
useJUnitPlatform {
includeTags('large-model')
}
systemProperty('radixor.test.modelId', runtimeModelId.get())
minHeapSize = '1g'
maxHeapSize = largeModelMaxHeap.get()
maxParallelForks = 1
forkEvery = 1
reports {
junitXml.required = true
html.required = true
}
doFirst {
logger.lifecycle("Runtime model integration uses model '{}' with maximum heap {}.",
systemProperties.get('radixor.test.modelId'), maxHeapSize)
}
}
def configureTaggedTestProfile = { String taskName, String includeTagsExpr, String excludeTagsExpr = null,
String taskDescription = null, String testNameExcludePatterns = null ->
tasks.register(taskName, Test) {
group = 'verification'
description = taskDescription
configureJUnitPlatformTags(delegate as Test, includeTagsExpr, excludeTagsExpr)
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
dependsOn(tasks.named('compileTestJava'))
if (testNameExcludePatterns != null && !testNameExcludePatterns.isBlank()) {
filter {
testNameExcludePatterns.split(',').each { String pattern ->
final def trimmedPattern = pattern.trim()
if (!trimmedPattern.isEmpty()) {
excludeTestsMatching(trimmedPattern)
}
}
}
}
minHeapSize = '1g'
maxHeapSize = '4g'
reports {
junitXml.required = true
html.required = true
}
}
}
configureTaggedTestProfile(
'ciSmoke',
'unit',
'slow',
'Fast feedback profile for unit tests with slow tests explicitly excluded.',
'org.egothor.stemmer.CompileIntegrationTest*'
)
configureTaggedTestProfile(
'ciCore',
'unit,trie,frequency-trie,property',
null,
'Focused profile for core trie behavior and trie-specific property checks.'
)
configureTaggedTestProfile(
'ciIntegration',
'integration',
'slow',
'Integration pipeline profile (loader/parser/CLI/IO end-to-end flows) excluding slow integration paths.'
)
configureTaggedTestProfile(
'ciSlow',
'slow',
null,
'Targeted profile for all slow tests (large dictionaries, long-running corpus validation, and heavy integration checks).'
)
configureTaggedTestProfile(
'ciCompat',
'compat,regression',
null,
'Compatibility profile guarding persisted artifact and compatibility regressions.'
)
configureTaggedTestProfile(
'ciRelease',
null,
'slow,large-model',
'Release-profile validation of all non-slow tests.',
'org.egothor.stemmer.CompileIntegrationTest*,org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*'
)
tasks.named('ciRelease', Test) {
dependsOn('prepareModelConsumerTestRepository')
systemProperty('radixor.consumer.repository',
layout.buildDirectory.dir('model-consumer-repository').get().asFile.absolutePath)
systemProperty('radixor.core.version', version.toString())
systemProperty('radixor.catalog.version', project(':models:standard').version.toString())
}
configureTaggedTestProfile(
'ciNightly',
'fuzz',
null,
'Nightly robustness profile with fuzz testing emphasis.'
)
tasks.register('ci') {
group = 'verification'
description = 'Runs the full enterprise CI profile set in sequence.'
dependsOn(tasks.named('ciSmoke'))
dependsOn(tasks.named('ciCore'))
dependsOn(tasks.named('ciIntegration'))
dependsOn(tasks.named('ciCompat'))
}
tasks.withType(Pmd).configureEach {
reports {
xml.required = true
html.required = true
}
}
tasks.named('jacocoTestReport', JacocoReport) {
dependsOn(tasks.named('test'))
classDirectories.setFrom(
files(sourceSets.main.output).asFileTree.matching {
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
exclude 'org/egothor/stemmer/DiacriticStripper*'
}
)
reports {
xml.required = true
csv.required = false
html.required = true
}
}
def registerJacocoProfileReport = { String reportTaskName, String sourceTaskName ->
tasks.register(reportTaskName, JacocoReport) {
group = 'verification'
description = "Generates Jacoco report for ${sourceTaskName} execution."
dependsOn(tasks.named(sourceTaskName))
classDirectories.setFrom(
files(sourceSets.main.output).asFileTree.matching {
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
exclude 'org/egothor/stemmer/DiacriticStripper*'
}
)
executionData.setFrom(
fileTree(layout.buildDirectory.dir('jacoco')) {
include "${sourceTaskName}.exec"
}
)
reports {
xml.required = true
csv.required = false
html.required = true
}
}
}
registerJacocoProfileReport('jacocoCiReleaseReport', 'ciRelease')
tasks.named('check') {
dependsOn(tasks.named('jacocoTestReport'))
// no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze'))
}
tasks.register('verifyCoreJarExcludesModels') {
group = 'verification'
description = 'Verifies that the root Radixor JAR contains no language dictionary bytes.'
dependsOn(tasks.named('jar'))
doLast {
File archive = tasks.named('jar', Jar).get().archiveFile.get().asFile
List<String> dictionaries = []
new java.util.zip.ZipFile(archive).withCloseable { zip ->
zip.entries().each { entry -> if (entry.name.endsWith('/stemmer.gz')) dictionaries.add(entry.name) }
}
if (!dictionaries.isEmpty()) {
throw new GradleException('The org.egothor:radixor JAR must not contain model data: ' + dictionaries)
}
}
}
tasks.register('verifyJavaLicenseHeaders') {
group = 'verification'
description = 'Verifies deterministic license classification for every maintained Java source file.'
inputs.file(layout.projectDirectory.file('gradle/java-license-header.txt'))
inputs.files(fileTree('src/main/java') { include '**/*.java' })
inputs.files(fileTree('src/test/java') { include '**/*.java' })
inputs.files(fileTree('src/jmh/java') { include '**/*.java' })
outputs.file(layout.buildDirectory.file('reports/license/java-license-headers.txt'))
doLast {
String canonicalHeader = layout.projectDirectory.file('gradle/java-license-header.txt')
.asFile.getText('UTF-8')
File canonicalSource = file('src/main/java/org/egothor/stemmer/CaseProcessingMode.java')
if (!canonicalSource.getText('UTF-8').startsWith(canonicalHeader)) {
throw new GradleException('CaseProcessingMode.java does not begin with the canonical Radixor license template.')
}
List<File> maintainedSources = files(
fileTree('src/main/java') { include '**/*.java' },
fileTree('src/test/java') { include '**/*.java' },
fileTree('src/jmh/java') { include '**/*.java' })
.files.toList().sort { File left, File right ->
relativePath(left) <=> relativePath(right)
}
List<String> classifications = []
List<String> failures = []
maintainedSources.each { File sourceFile ->
String relative = relativePath(sourceFile)
String content = sourceFile.getText('UTF-8')
if (content.startsWith(canonicalHeader)) {
classifications.add("RADIXOR_CANONICAL_HEADER ${relative}")
return
}
String leadingNotice = ''
if (content.startsWith('/*')) {
int closingIndex = content.indexOf('*/')
if (closingIndex >= 0) {
leadingNotice = content.substring(0, closingIndex + 2)
}
} else if (content.startsWith('//')) {
leadingNotice = content.readLines().takeWhile { String line ->
line.startsWith('//') || line.isBlank()
}.join('\n')
}
String remainder = content.substring(leadingNotice.length()).stripLeading()
boolean duplicateNotice = !leadingNotice.isEmpty()
&& (remainder.startsWith('/*') || remainder.startsWith('//'))
boolean historicalRadixor = leadingNotice =~ /(?s)Copyright \(C\) \d{4}(?:-\d{4})?, Leo Galambos/
&& leadingNotice.contains('All rights reserved.')
&& leadingNotice.contains('Redistribution and use in source and binary forms')
&& leadingNotice.contains('THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS')
boolean thirdPartyOrProvenance = leadingNotice =~ /(?is)(SPDX-License-Identifier|Licensed under|MIT License|Apache License|Permission is hereby granted|Original source|Adapted from|Ported from|Source:\s*\S)/
if (duplicateNotice) {
classifications.add("AMBIGUOUS_AUTHORSHIP ${relative}")
failures.add("${relative}: duplicate leading comment blocks")
} else if (historicalRadixor) {
classifications.add("RADIXOR_HISTORICAL_HEADER ${relative}")
} else if (thirdPartyOrProvenance) {
classifications.add("THIRD_PARTY_OR_PROVENANCE_HEADER ${relative}")
} else {
classifications.add("AMBIGUOUS_AUTHORSHIP ${relative}")
failures.add("${relative}: no recognized governing license or provenance header")
}
}
File report = layout.buildDirectory.file('reports/license/java-license-headers.txt').get().asFile
report.parentFile.mkdirs()
report.setText(classifications.join('\n') + '\n', 'UTF-8')
if (!failures.isEmpty()) {
throw new GradleException('Maintained Java license verification failed: '
+ failures.sort().join(', '))
}
}
}
tasks.register('verifyAllDefaultModels') {
group = 'verification'
description = 'Verifies that every language default model project is configured.'
dependsOn(defaultModelProjects().collect { Project modelProject ->
modelProject.path + ':verifyModelDescriptor'
})
}
tasks.register('verifyAllModels') {
group = 'verification'
description = 'Runs complete validation and artifact verification for every independently versioned model module.'
dependsOn(modelProjects().collect { Project modelProject -> modelProject.tasks.named('check') })
}
tasks.register('verifyJmhModelClasspath') {
group = 'verification'
description = 'Verifies that JMH receives each individual model JAR exactly once and embeds no dictionary.'
dependsOn(tasks.named('jmhJar'))
dependsOn(modelProjects().collect { Project modelProject -> modelProject.tasks.named('jar') })
outputs.file(layout.buildDirectory.file('reports/models/jmh-model-classpath.txt'))
doLast {
List<File> modelJars = configurations.jmhRuntimeClasspath.files.findAll { File dependency ->
dependency.name.startsWith('radixor-model-') && dependency.name.endsWith('.jar')
}.sort { File left, File right -> left.name <=> right.name }
List<String> expectedPrefixes = modelProjects().collect { Project modelProject ->
"radixor-model-${modelProject.name}-"
}
expectedPrefixes.each { String prefix ->
List<File> matches = modelJars.findAll { File dependency -> dependency.name.startsWith(prefix) }
if (matches.size() != 1) {
throw new GradleException("JMH must resolve exactly one model JAR with prefix ${prefix}; resolved ${matches}.")
}
}
if (modelJars.any { File dependency -> dependency.name.contains('benchmark-pack') }) {
throw new GradleException('JMH must not resolve a benchmark-pack artifact.')
}
File executable = tasks.named('jmhJar', Jar).get().archiveFile.get().asFile
if (!zipTree(executable).matching { include '**/stemmer.gz' }.isEmpty()) {
throw new GradleException('The JMH executable JAR must not embed model dictionaries.')
}
File report = layout.buildDirectory.file('reports/models/jmh-model-classpath.txt').get().asFile
report.parentFile.mkdirs()
report.setText(modelJars.collect { File dependency -> dependency.name }.join('\n') + '\n', 'UTF-8')
}
}
tasks.named('prepareModelConsumerTestRepository') {
coreVersion = version.toString()
catalogVersion = project(':models:standard').version.toString()
modelVersions = modelProjects().collectEntries { Project modelProject ->
final String modelVersion = providers.gradleProperty('modelReleaseVersion')
.orElse(providers.fileContents(modelProject.layout.projectDirectory.file('model-version.txt'))
.asText.map(String::trim))
.get()
[(modelProject.name): modelVersion]
}
corePom = layout.file(tasks.named('generatePomFileForMavenJavaPublication').map { it.destination })
coreJar = tasks.named('jar', Jar).flatMap { it.archiveFile }
modelPoms.from(modelProjects().collect { Project modelProject ->
modelProject.tasks.named('generatePomFileForModelPublication').map { it.destination }
})
modelJars.from(modelProjects().collect { Project modelProject ->
modelProject.tasks.named('jar', Jar).flatMap { it.archiveFile }
})
standardPom = layout.file(project(':models:standard').tasks.named('generatePomFileForStandardPublication')
.map { it.destination })
bomPom = layout.file(project(':models:bom').tasks.named('generatePomFileForBomPublication')
.map { it.destination })
repositoryDirectory = layout.buildDirectory.dir('model-consumer-repository')
}
def rawModelCatalogRepository = layout.buildDirectory.dir('model-catalog-staging-repository')
def preparedModelCatalogBundleInput = layout.buildDirectory.dir('model-catalog-bundle-input')
def cleanModelCatalogStaging = tasks.register('cleanModelCatalogStaging', Delete) {
group = 'publishing'
description = 'Cleans the isolated model catalog Maven staging repository.'
delete(rawModelCatalogRepository)
}
gradle.projectsEvaluated {
project(':models:standard').tasks.named('publishStandardPublicationToCatalogStagingRepository') {
dependsOn(cleanModelCatalogStaging)
}
project(':models:bom').tasks.named('publishBomPublicationToCatalogStagingRepository') {
dependsOn(cleanModelCatalogStaging)
}
}
def prepareModelCatalogReleaseCandidate = tasks.named('prepareModelCatalogReleaseCandidate') {
dependsOn(project(':models:standard').tasks.named('check'))
dependsOn(project(':models:bom').tasks.named('check'))
dependsOn(':models:standard:publishStandardPublicationToCatalogStagingRepository')
dependsOn(':models:bom:publishBomPublicationToCatalogStagingRepository')
rawRepositoryDirectory = rawModelCatalogRepository
preparedBundleDirectory = preparedModelCatalogBundleInput
catalogVersion = project(':models:standard').version.toString()
}
def modelCatalogCentralBundle = tasks.register('modelCatalogCentralBundle', Zip) {
group = 'publishing'
description = 'Builds the local POM-only model catalog bundle without remote publication.'
dependsOn(prepareModelCatalogReleaseCandidate)
from(preparedModelCatalogBundleInput)
destinationDirectory = layout.buildDirectory.dir('model-catalog-release-candidate')
archiveFileName = "radixor-models-catalog-${project(':models:standard').version}-central-bundle.zip"
doFirst {
File preparedInput = preparedModelCatalogBundleInput.get().asFile
if (!preparedInput.isDirectory() || preparedInput.listFiles() == null || preparedInput.listFiles().length == 0) {
throw new GradleException("The prepared model catalog bundle input is missing or empty: ${preparedInput}.")
}
if (providers.environmentVariable('GITHUB_REF_TYPE').orNull == 'tag'
&& (providers.environmentVariable('SIGNING_KEY').orNull?.isBlank() != false
|| providers.environmentVariable('SIGNING_PASSWORD').orNull?.isBlank() != false)) {
throw new GradleException('A tagged model catalog release requires SIGNING_KEY and SIGNING_PASSWORD.')
}
}
}
def publishedModelVersions = modelProjects().collect { Project modelProject ->
modelProject.file('model-version.txt').getText('UTF-8').trim()
}.toSet()
if (publishedModelVersions.size() != 1) {
throw new GradleException("The catalog verifier requires one common model version; found ${publishedModelVersions}.")
}
tasks.named('verifyModelCatalogReleaseCandidate') {
bundleFile = modelCatalogCentralBundle.flatMap { Zip archive -> archive.archiveFile }
reportFile = layout.buildDirectory.file('reports/models/catalog-release-candidate.txt')
catalogVersion = project(':models:standard').version.toString()
modelVersion = publishedModelVersions.first()
defaultModelIds = defaultModelProjects().collect { Project modelProject -> modelProject.name }
allModelIds = modelProjects().collect { Project modelProject -> modelProject.name }
}
tasks.register('verifyArtifactSizes') {
group = 'verification'
description = 'Reports artifact sizes and rejects dictionary bytes in core.'
dependsOn(tasks.named('verifyCoreJarExcludesModels'))
doLast {
File archive = tasks.named('jar', Jar).get().archiveFile.get().asFile
println('org.egothor:radixor:' + version + ' ' + archive.length() + ' bytes')
}
}
tasks.named('check') {
dependsOn(tasks.named('verifyCoreJarExcludesModels'))
dependsOn(tasks.named('verifyAllDefaultModels'))
dependsOn(tasks.named('verifyJmhModelClasspath'))
dependsOn(tasks.named('verifyJavaLicenseHeaders'))
}
def modelCatalogText = providers.provider {
StringBuilder output = new StringBuilder()
output.append('| Model ID | Language | Default | Coordinates | Version | Source | Repository | Source version | Revision | Revision status | License | Attribution | SHA-256 | Bytes |\n')
output.append('|---|---|---:|---|---:|---|---|---|---|---|---|---|---|---:|\n')
modelProjects().each { Project modelProject ->
String script = modelProject.file('build.gradle').getText('UTF-8')
def value = { String key ->
def matcher = script =~ /(?m)^\s*${key}\s*=\s*(?:'([^']+)'|([^\s]+))\s*$/
if (!matcher.find()) return 'unavailable'
return matcher.group(1) != null ? matcher.group(1) : matcher.group(2)
}
File input = modelProject.file('src/modelInput/stemmer.gz')
String checksum = java.security.MessageDigest.getInstance('SHA-256').digest(input.bytes).encodeHex().toString()
output.append('| ').append(modelProject.name)
.append(' | ').append(value('language'))
.append(' | ').append(value('defaultModel'))
.append(' | org.egothor:radixor-model-').append(modelProject.name)
.append(' | ').append(modelProject.file('model-version.txt').text.trim())
.append(' | ').append(value('sourceName'))
.append(' | ').append(value('sourceRepository'))
.append(' | ').append(value('sourceVersion'))
.append(' | ').append(value('sourceRevision'))
.append(' | ').append(value('sourceRevisionStatus'))
.append(' | ').append(value('sourceLicense'))
.append(' | ').append(value('sourceAttribution'))
.append(' | ').append(checksum)
.append(' | ').append(input.length()).append(' |\n')
}
return output.toString()
}
def modelCatalogDocumentationInputs = files(modelProjects().collectMany { Project modelProject ->
[
modelProject.file('build.gradle'),
modelProject.file('model-version.txt'),
modelProject.file('src/modelInput/stemmer.gz')
]
})
tasks.register('generateModelCatalogDocumentation') {
group = 'documentation'
description = 'Generates the deterministic model catalog in the build documentation staging tree.'
inputs.files(modelCatalogDocumentationInputs)
outputs.file(layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md'))
doLast {
File catalog = layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md').get().asFile
catalog.parentFile.mkdirs()
catalog.setText('# Published Stemmer Model Catalog\n\n' + modelCatalogText.get(), 'UTF-8')
}
}
tasks.register('publishModelCatalogDocumentation') {
group = 'documentation'
description = 'Updates the checked-in model catalog used by a direct local MkDocs invocation.'
dependsOn(tasks.named('generateModelCatalogDocumentation'))
inputs.file(layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md'))
outputs.file(layout.projectDirectory.file('docs/stemmer-model-catalog.md'))
doLast {
File generated = layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md').get().asFile
File published = layout.projectDirectory.file('docs/stemmer-model-catalog.md').asFile
published.setText(generated.getText('UTF-8'), 'UTF-8')
}
}
tasks.register('prepareMkDocsSource', Sync) {
group = 'documentation'
description = 'Stages maintained documentation, generated catalog, and MkDocs configuration under build/.'
dependsOn(modelProjects().collect { Project modelProject -> modelProject.path + ':verifyModelDescriptor' })
inputs.files(modelCatalogDocumentationInputs)
into(layout.buildDirectory.dir('mkdocs-source'))
from(layout.projectDirectory.dir('docs'))
doLast {
File catalog = layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md').get().asFile
catalog.setText('# Published Stemmer Model Catalog\n\n' + modelCatalogText.get(), 'UTF-8')
File buildsPage = layout.buildDirectory.file('mkdocs-source/builds.md').get().asFile
buildsPage.setText('# Historical Builds\n\nThe Pages publication workflow replaces this staging placeholder with the retained build index.\n', 'UTF-8')
File configuration = layout.buildDirectory.file('mkdocs/mkdocs.yml').get().asFile
configuration.parentFile.mkdirs()
String configurationText = layout.projectDirectory.file('mkdocs.yml').asFile.getText('UTF-8')
.replace('custom_dir: docs/overrides', 'custom_dir: ../mkdocs-source/overrides')
configuration.setText(configurationText
+ '\ndocs_dir: ../mkdocs-source\nsite_dir: ../mkdocs-site\n', 'UTF-8')
}
}
tasks.register('verifyModelCatalogDocumentation') {
group = 'verification'
description = 'Validates model metadata and both the checked-in and staged MkDocs catalogs.'
dependsOn(tasks.named('prepareMkDocsSource'))
dependsOn(tasks.named('verifyAllDefaultModels'))
doLast {
File catalog = layout.buildDirectory.file('mkdocs-source/stemmer-model-catalog.md').get().asFile
String expected = '# Published Stemmer Model Catalog\n\n' + modelCatalogText.get()
if (!catalog.isFile() || catalog.getText('UTF-8') != expected) {
throw new GradleException('The staged model catalog is missing or nondeterministic.')
}
File publishedCatalog = layout.projectDirectory.file('docs/stemmer-model-catalog.md').asFile
if (!publishedCatalog.isFile() || publishedCatalog.getText('UTF-8') != expected) {
throw new GradleException(
'The checked-in model catalog is stale; run ./gradlew publishModelCatalogDocumentation.')
}
List<String> identifiers = modelProjects().collect { Project modelProject -> modelProject.name }
if (identifiers != identifiers.sort()) {
throw new GradleException('Published model projects are not in deterministic model-ID order.')
}
identifiers.each { String identifier ->
if (!expected.contains('org.egothor:radixor-model-' + identifier)) {
throw new GradleException('The staged model catalog omits published model ' + identifier + '.')
}
}
if (!layout.buildDirectory.file('mkdocs/mkdocs.yml').get().asFile.isFile()) {
throw new GradleException('The staged MkDocs configuration is missing.')
}
}
}
tasks.named('check') {
dependsOn(tasks.named('verifyModelCatalogDocumentation'))
}
tasks.named('cyclonedxDirectBom') {
includeConfigs = ['runtimeClasspath', 'compileClasspath']
skipConfigs = ['testRuntimeClasspath', 'testCompileClasspath', 'jmh.*', 'mockitoAgent']
includeBomSerialNumber = true
includeLicenseText = false
includeMetadataResolution = true
includeBuildSystem = true
jsonOutput.set(sbomReportsDirectory.map { it.file('radixor-sbom.json') })
xmlOutput.set(sbomReportsDirectory.map { it.file('radixor-sbom.xml') })
}
subprojects {
tasks.matching { Task candidate -> candidate.name == 'cyclonedxDirectBom' }.configureEach {
enabled = false
description = 'Disabled because the root project exclusively owns CycloneDX SBOM generation.'
}
}
pitest {
pitestVersion = '1.22.1'
junit5PluginVersion = '1.2.3'
targetClasses = [
'org.egothor.stemmer.*',
'org.egothor.stemmer.trie.*'
]
targetTests = [
'org.egothor.stemmer.*Test',
'org.egothor.stemmer.trie.*Test'
]
excludedClasses = [
'org.egothor.stemmer.Compile*',
'org.egothor.stemmer.StemmerPatchTrieLoader*',
'org.egothor.stemmer.StemmerKnowledgeExperiment*',
'org.egothor.stemmer.StemmerKnowledgeExperimentCli*'
]
excludedTestClasses = [
'org.egothor.stemmer.CompileIntegrationTest',
'org.egothor.stemmer.StemmerPatchTrieLoaderTest',
'org.egothor.stemmer.StemmerKnowledgeExperimentTest',
// These integration tests require dedicated Gradle task wiring and must not run in PIT worker JVMs.
'org.egothor.stemmer.FullRuntimeModelIntegrationTest',
'org.egothor.stemmer.ModelDependencyResolutionTest'
]
outputFormats = ['XML', 'HTML']
timestampedReports = false
exportLineCoverage = true
failWhenNoMutations = true
threads = Math.max(1, Runtime.runtime.availableProcessors().intdiv(2))
}
application {
mainClass = 'org.egothor.stemmer.Compile'
applicationName = 'radixor'
executableDir = 'bin'
}
tasks.register('stemmerKnowledgeExperiment', JavaExec) {
group = 'application'
description = 'Runs the stemmer knowledge evaluation experiment.'
classpath = sourceSets.main.runtimeClasspath
mainClass = 'org.egothor.stemmer.StemmerKnowledgeExperimentCli'
}
distributions {
main {
distributionBaseName = 'radixor'
contents {
from('README.md') {
into ''
}
from('LICENSE') {
into ''
}
from('LICENSE-stemmer-data') {
into ''
}
from('docs') {
into 'docs'
include '**/*.md'
}
from(layout.buildDirectory.dir('generated/release-notes')) {
into ''
include 'CHANGELOG.md'
}
}
}
}
tasks.named('startScripts') {
applicationName = 'radixor'
}
tasks.named('distZip', Zip) {
archiveBaseName = 'radixor'
archiveClassifier = 'bin'
}
tasks.named('distTar') {
enabled = false
}
jmh {
jmhVersion = '1.37'
includeTests = false
warmupIterations = 3
iterations = 5
fork = 1
benchmarkMode = ['avgt']
timeUnit = 'ns'
resultFormat = 'CSV'
resultsFile = benchmarkReportsDirectory.map { it.file('jmh-results.csv').asFile }.get()
humanOutputFile = benchmarkReportsDirectory.map { it.file('jmh-results.txt').asFile }.get()
duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
if (jmhIncludesProperty.isPresent()) {
includes = [jmhIncludesProperty.get()]
}
}
tasks.named('jmh') {
group = 'verification'
description = 'Runs JMH benchmarks for the Radixor algorithmic core and external stemmer comparison suites.'
}
tasks.named('jmhJar', Jar) {
exclude 'META-INF/radixor/models.index'
exclude 'META-INF/radixor/models/**'
exclude 'org/egothor/stemmer/models/**'
exclude 'META-INF/LICENSES/**'
}
apply from: 'gradle/lucene-benchmarks.gradle'
tasks.register('regressionArtifactGenerator', JavaExec) {
group = 'verification'
description = 'Generates deterministic compiled trie regression artifacts.'
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.egothor.stemmer.RegressionArtifactGenerator'
if (project.hasProperty('regressionInput')) {
args '--input', project.property('regressionInput').toString()
}
if (project.hasProperty('regressionOutput')) {
args '--output', project.property('regressionOutput').toString()
}
if (project.hasProperty('regressionStoreOriginal')) {
args '--store-original', project.property('regressionStoreOriginal').toString()
}
if (project.hasProperty('regressionReductionMode')) {
args '--reduction-mode', project.property('regressionReductionMode').toString()
}
}
tasks.register('stemmingQuality', JavaExec) {
group = 'verification'
description = 'Evaluates pairwise over-stemming and under-stemming against registered model dictionary groups.'
dependsOn(tasks.named('testClasses'))
dependsOn(tasks.named('jmhClasses'))
dependsOn(tasks.named('prepareBenchmarkModelInputs'))
classpath = files(sourceSets.test.runtimeClasspath, configurations.stemmingQualityJmhRuntime)
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityApplication'
args layout.buildDirectory.dir('reports/stemming-quality').get().asFile.absolutePath,
layout.buildDirectory.dir('generated/benchmark-model-inputs').get().asFile.absolutePath,
providers.gradleProperty('stemmingQualityLanguage').getOrElse(''),
providers.gradleProperty('stemmingQualityStemmer').getOrElse(''),
providers.gradleProperty('stemmingQualityMode').getOrElse(''),
providers.gradleProperty('stemmingQualityOutputPolicy').getOrElse(''),
providers.gradleProperty('stemmingQualityRankMetric').getOrElse('PAIRWISE_F05'),
providers.gradleProperty('stemmingQualityAudit').getOrElse('false'),
providers.gradleProperty('stemmingQualityAuditLimit').getOrElse('25')
maxHeapSize = '6g'
}
tasks.register('benchmarkCorpusReport', JavaExec) {
group = 'verification'
description = 'Reports corpus and preferred patch-command counts for every default model.'
dependsOn(tasks.named('jmhClasses'))
classpath = files(sourceSets.jmh.runtimeClasspath, configurations.stemmingQualityJmhRuntime)
mainClass = 'org.egothor.stemmer.benchmark.BenchmarkCorpusReportApplication'
args layout.buildDirectory.file('reports/jmh/benchmark-corpora.csv').get().asFile.absolutePath
maxHeapSize = '6g'
}
tasks.register('writeJmhRuntimeClasspath') {
group = 'verification'
description = 'Writes the complete modular JMH runtime classpath for isolated direct JMH execution.'
dependsOn(tasks.named('jmhJar'))
dependsOn(modelProjects().collect { Project modelProject -> modelProject.tasks.named('jar') })
outputs.file(layout.buildDirectory.file('reports/jmh/jmh-runtime-classpath.txt'))
doLast {
File report = layout.buildDirectory.file('reports/jmh/jmh-runtime-classpath.txt').get().asFile
report.parentFile.mkdirs()
File executable = tasks.named('jmhJar', Jar).get().archiveFile.get().asFile
report.setText(executable.absolutePath + File.pathSeparator
+ sourceSets.jmh.runtimeClasspath.asPath + System.lineSeparator(), 'UTF-8')
}
}
tasks.register('prepareBenchmarkModelInputs', Sync) {
group = 'verification'
description = 'Prepares default model inputs for JMH and quality evaluation without changing source data.'
into(layout.buildDirectory.dir('generated/benchmark-model-inputs'))
defaultModelProjects().each { Project modelProject ->
String languageDirectory = modelProject.name == 'pl-pl-unimorph'
? 'pl_pl'
: modelProject.name.replace('-default', '').replace('-', '_')
from(modelProject.file('src/modelInput/stemmer.gz')) {
into(languageDirectory)
}
}
}
tasks.register('publishStemmingQualityDocumentation', JavaExec) {
group = 'documentation'
description = 'Publishes validated complete stemming-quality results on the language benchmark pages.'
dependsOn(tasks.named('testClasses'))
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityDocumentationPublisher'
args layout.buildDirectory.file('reports/stemming-quality/stemming-quality.csv').get().asFile.absolutePath,
layout.projectDirectory.dir('docs').asFile.absolutePath,
'update'
doFirst {
if (!file("$buildDir/reports/stemming-quality/stemming-quality.csv").isFile()) {
throw new GradleException('A complete stemming-quality CSV is required. Run stemmingQuality only when no validated complete report is available.')
}
}
}
tasks.register('verifyStemmingQualityDocumentation', JavaExec) {
group = 'verification'
description = 'Verifies published language-page quality tables against the checked-in authoritative CSV.'
dependsOn(tasks.named('testClasses'))
classpath = sourceSets.test.runtimeClasspath
mainClass = 'org.egothor.stemmer.benchmark.quality.StemmingQualityDocumentationPublisher'
args layout.projectDirectory.file('docs/benchmarks/data/stemming-quality.csv').asFile.absolutePath,
layout.projectDirectory.dir('docs').asFile.absolutePath,
'verify'
}
tasks.named('check') {
dependsOn(tasks.named('verifyStemmingQualityDocumentation'))
}
tasks.register('verifyStemmingQualitySourceSets') {
group = 'verification'
description = 'Verifies the production, JMH, and standard-test ownership of stemming-quality infrastructure.'
doLast {
if (sourceSets.findByName('stemmingQualityTest') != null || file('src/stemmingQualityTest').exists()) {
throw new GradleException('The obsolete stemmingQualityTest source set or directory still exists.')
}
if (!file('src/jmh/java/org/egothor/stemmer/benchmark/QualityStemmerMatrix.java').isFile()) {
throw new GradleException('The authoritative JMH stemmer matrix is not in src/jmh.')
}
if (!file('src/test/java/org/egothor/stemmer/benchmark/quality/StemmingQualityApplication.java').isFile()) {
throw new GradleException('The stemming-quality evaluator is not in the standard test source set.')
}
}
}
tasks.register('verifyProductionJarExcludesStemmingQuality') {
group = 'verification'
description = 'Verifies that analytical stemming-quality classes are absent from the production JAR.'
dependsOn(tasks.named('jar'))
doLast {
final File archive = tasks.named('jar').get().archiveFile.get().asFile
final def forbidden = zipTree(archive).matching { include '**/benchmark/**' }.files
if (!forbidden.isEmpty()) {
throw new GradleException("Production JAR contains analytical stemming-quality classes: ${forbidden}")
}
}
}
tasks.register('printDependencyCheckNvdConfig') {
doLast {
System.out.println("NVD API key present: " + (nvdApiKey != null && !nvdApiKey.isBlank()))
}
}
tasks.named('dependencyCheckAnalyze') {
dependsOn(tasks.named('printDependencyCheckNvdConfig'))
}
javadoc {
failOnError = false
options.addStringOption('Xdoclint:all,-missing', '-quiet')
options.addBooleanOption('html5', true)
options.tags('apiNote:a:API Note:')
options.tags('implSpec:a:Implementation Requirements:')
options.tags('implNote:a:Implementation Note:')
options.tags('param')
options.tags('return')
options.tags('throws')
options.tags('since')
options.tags('version')
options.tags('serialData')
options.tags('factory')
options.tags('see')
options.use = true
options.author = true
options.version = true
options.windowTitle = 'Radixor - Egothor Stemmer'
options.docTitle = 'Radixor - Egothor Stemmer API'
options.overview = file('src/main/javadoc/overview.html')
options.bottom = """
<div class="legal-copy">
&copy; 2026 Egothor
<br/>
Licensed under <a href="https://github.com/leogalambos/Radixor/blob/main/LICENSE">BSD-3-Clause</a>
</div>
"""
options.links('https://docs.oracle.com/en/java/javase/21/docs/api/')
options.group('Core Stemming API', 'org.egothor.stemmer')
options.group('Trie Infrastructure', 'org.egothor.stemmer.trie')
source = sourceSets.main.allJava
}
apply from: 'gradle/snowball-benchmarks.gradle'
apply from: 'gradle/paicehusk-benchmarks.gradle'
apply from: 'gradle/opennlp-benchmarks.gradle'
apply from: 'gradle/hunspell-benchmarks.gradle'
apply from: 'gradle/cistem-benchmarks.gradle'
apply from: 'gradle/python.gradle'
gradle.taskGraph.whenReady { taskGraph ->
def banner = """
\u001B[34m
8888888888 .d8888b. .d88888b. 88888888888 888 888 .d88888b. 8888888b.
888 d88P Y88b d88P" "Y88b 888 888 888 d88P" "Y88b 888 Y88b
888 888 888 888 888 888 888 888 888 888 888 888
8888888 888 888 888 888 8888888888 888 888 888 d88P
888 888 88888 888 888 888 888 888 888 888 8888888P"
888 888 888 888 888 888 888 888 888 888 888 T88b
888 Y88b d88P Y88b. .d88P 888 888 888 Y88b. .d88P 888 T88b
8888888888 "Y8888P88 "Y88888P" 888 888 888 "Y88888P" 888 T88b
\u001B[36m
Project : ${project.name}
Version : ${project.version}
\u001B[0m
"""
println banner
}