feat!: modularize stemmer models and release infrastructure

Move bundled stemmer dictionaries from the core artifact into independently
versioned model modules. Add model discovery and explicit model-loading APIs,
a standard model aggregate, a model BOM, and dedicated model and catalog
release workflows.

Add full PoliMorf integration, model provenance and licensing validation,
streaming model-input verification, strict dependency verification, consumer
resolution tests, Configuration Cache compatibility, and expanded JMH,
quality, documentation, and release checks.

Upgrade the CycloneDX and JMH Gradle plugins and remove Gradle 10 and Java
compiler deprecations.

BREAKING CHANGE: The core Radixor artifact no longer contains bundled stemmer
dictionaries. Applications must add the required model artifacts, the standard
model aggregate, or model dependencies managed through the Radixor model BOM.
This commit is contained in:
2026-07-22 23:33:28 +02:00
parent 9c5b9e331b
commit e7800b29c9
250 changed files with 7297 additions and 905 deletions

View File

@@ -0,0 +1,21 @@
package org.egothor.radixor
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.tasks.Classpath
import org.gradle.process.CommandLineArgumentProvider
import javax.inject.Inject
abstract class MockitoAgentArgumentProvider implements CommandLineArgumentProvider {
@Classpath
abstract ConfigurableFileCollection getAgentClasspath()
@Inject
MockitoAgentArgumentProvider() {
}
@Override
Iterable<String> asArguments() {
return ["-javaagent:${agentClasspath.singleFile.absolutePath}"]
}
}

View File

@@ -0,0 +1,105 @@
package org.egothor.radixor
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.stream.Stream
/** Builds the isolated Maven-layout repository used by consumer resolution tests. */
abstract class PrepareModelConsumerRepositoryTask extends DefaultTask {
@Input abstract Property<String> getCoreVersion()
@Input abstract Property<String> getCatalogVersion()
@Input abstract MapProperty<String, String> getModelVersions()
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
abstract RegularFileProperty getCorePom()
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
abstract RegularFileProperty getCoreJar()
@InputFiles @PathSensitive(PathSensitivity.RELATIVE)
abstract ConfigurableFileCollection getModelPoms()
@InputFiles @PathSensitive(PathSensitivity.RELATIVE)
abstract ConfigurableFileCollection getModelJars()
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
abstract RegularFileProperty getStandardPom()
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
abstract RegularFileProperty getBomPom()
@OutputDirectory
abstract DirectoryProperty getRepositoryDirectory()
/** Creates the repository using only declared task state and Java file APIs. */
@TaskAction
void prepareRepository() {
final Path repository = repositoryDirectory.get().asFile.toPath()
deleteTree(repository)
Files.createDirectories(repository)
install(repository, 'radixor', coreVersion.get(), corePom.get().asFile.toPath(), coreJar.get().asFile.toPath())
final Map<String, Path> pomsByModel = indexModelFiles(modelPoms.files)
final Map<String, Path> jarsByModel = indexModelFiles(modelJars.files)
modelVersions.get().toSorted().each { String modelId, String modelVersion ->
final Path pom = pomsByModel.get(modelId)
final Path jar = jarsByModel.get(modelId)
if (pom == null || jar == null) {
throw new GradleException("Missing generated publication input for model ${modelId}.")
}
PrepareModelConsumerRepositoryTask.install(
repository, "radixor-model-${modelId}", modelVersion, pom, jar)
}
install(repository, 'radixor-models-standard', catalogVersion.get(), standardPom.get().asFile.toPath(), null)
install(repository, 'radixor-models-bom', catalogVersion.get(), bomPom.get().asFile.toPath(), null)
}
private static Map<String, Path> indexModelFiles(final Set<File> files) {
final Map<String, Path> indexed = [:]
files.each { File file ->
Path cursor = file.toPath().toAbsolutePath().parent
while (cursor != null && cursor.fileName.toString() != 'build') cursor = cursor.parent
if (cursor == null || cursor.parent == null) {
throw new GradleException("Cannot determine model ID from generated input ${file}.")
}
final String modelId = cursor.parent.fileName.toString()
if (indexed.put(modelId, file.toPath()) != null) {
throw new GradleException("Duplicate generated publication input for model ${modelId}.")
}
}
return indexed
}
private static void install(final Path repository, final String artifactId, final String version,
final Path pom, final Path jar) {
final Path module = repository.resolve("org/egothor/${artifactId}/${version}")
Files.createDirectories(module)
Files.copy(pom, module.resolve("${artifactId}-${version}.pom"), StandardCopyOption.REPLACE_EXISTING)
if (jar != null) {
Files.copy(jar, module.resolve("${artifactId}-${version}.jar"), StandardCopyOption.REPLACE_EXISTING)
}
}
private static void deleteTree(final Path directory) {
if (!Files.exists(directory)) return
Files.walk(directory).withCloseable { Stream<Path> paths ->
paths.sorted(Comparator.reverseOrder()).forEach(Files::delete)
}
}
}

View File

@@ -0,0 +1,108 @@
package org.egothor.radixor
import org.gradle.api.DefaultTask
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.security.MessageDigest
import java.util.stream.Stream
/** Generates one model's deterministic resource tree without retaining Project state. */
abstract class PrepareModelResourcesTask extends DefaultTask {
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getDictionaryFile()
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getVersionFile()
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getLicenseFile()
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getNoticeFile()
@Input abstract Property<Boolean> getShareAlike()
@Input abstract MapProperty<String, String> getDescriptorValues()
@OutputDirectory abstract DirectoryProperty getGeneratedDirectory()
/** Copies bounded inputs and writes descriptor and index files. */
@TaskAction
void prepareResources() {
final Path generated = generatedDirectory.get().asFile.toPath()
deleteTree(generated)
final Map<String, String> values = descriptorValues.get()
final String id = values['model.id']
final String resource = "org/egothor/stemmer/models/${id}/stemmer.gz"
final Path dictionaryTarget = generated.resolve(resource)
Files.createDirectories(dictionaryTarget.parent)
Files.copy(dictionaryFile.get().asFile.toPath(), dictionaryTarget, StandardCopyOption.REPLACE_EXISTING)
final Path descriptor = generated.resolve("META-INF/radixor/models/${id}.properties")
Files.createDirectories(descriptor.parent)
Files.writeString(descriptor, descriptorText(values,
versionFile.get().asFile.getText('UTF-8').trim(), resource, sha256(dictionaryFile.get().asFile)))
final Path index = generated.resolve('META-INF/radixor/models.index')
Files.createDirectories(index.parent)
Files.writeString(index, "META-INF/radixor/models/${id}.properties\n")
if (shareAlike.get()) {
final Path notice = generated.resolve("META-INF/NOTICE/${id}-data.txt")
Files.createDirectories(notice.parent)
Files.copy(noticeFile.get().asFile.toPath(), notice, StandardCopyOption.REPLACE_EXISTING)
} else {
final Path license = generated.resolve('META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt')
Files.createDirectories(license.parent)
Files.copy(licenseFile.get().asFile.toPath(), license, StandardCopyOption.REPLACE_EXISTING)
}
}
private static String descriptorText(final Map<String, String> value, final String version,
final String resource, final String checksum) {
return """model.id=${value['model.id']}
model.version=${version}
model.language=${value['model.language']}
model.displayName=${value['model.displayName']}
model.resource=${resource}
model.default=${value['model.default']}
model.format=radixor-dictionary-tsv-gzip
model.formatVersion=1
model.sha256=${checksum}
model.rightToLeft=${['FA_IR', 'HE_IL', 'YI'].contains(value['model.language'])}
model.caseProcessing=LOWERCASE_WITH_LOCALE_ROOT
model.diacriticProcessing=AS_IS
model.storeOriginal=true
source.name=${value['source.name']}
source.version=${value['source.version']}
source.project=${value['source.project']}
source.repository=${value['source.repository']}
source.dataset=${value['source.dataset']}
source.revision=${value['source.revision']}
source.revisionStatus=${value['source.revisionStatus']}
source.license=${value['source.license']}
source.licenseUri=${value['source.licenseUri']}
source.attribution=${value['source.attribution']}
source.verificationDate=${value['source.verificationDate']}
transformations.summary=${value['transformations.summary']}
compiler.radixorVersion=3.x
compiler.radixorCommit=unavailable
statistics.groups=unavailable
statistics.forms=unavailable
"""
}
private static String sha256(final File file) {
return MessageDigest.getInstance('SHA-256').digest(file.bytes)
.collect { byte value -> String.format('%02x', value & 0xff) }.join()
}
private static void deleteTree(final Path directory) {
if (!Files.exists(directory)) return
Files.walk(directory).withCloseable { Stream<Path> paths ->
paths.sorted(Comparator.reverseOrder()).forEach(Files::delete)
}
}
}

View File

@@ -0,0 +1,16 @@
package org.egothor.radixor
import org.gradle.api.Plugin
import org.gradle.api.Project
/** Exposes typed repository build-support tasks to the root build. */
final class RadixorBuildSupportPlugin implements Plugin<Project> {
/** Registers build-support tasks without inspecting project state during execution. */
@Override
void apply(final Project project) {
project.tasks.register('prepareModelConsumerTestRepository', PrepareModelConsumerRepositoryTask) {
group = 'verification'
description = 'Creates an isolated local Maven repository for model dependency-resolution integration tests.'
}
}
}

View File

@@ -0,0 +1,73 @@
package org.egothor.radixor
import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.Property
import javax.inject.Inject
/** Declarative configuration for one independently published Radixor model. */
abstract class RadixorModelExtension {
/** Stable model identifier. */
abstract Property<String> getModelId()
/** Radixor language enum constant. */
abstract Property<String> getLanguage()
/** Human-readable model name. */
abstract Property<String> getDisplayName()
/** Whether this is the documented default for its language. */
abstract Property<Boolean> getDefaultModel()
/** Source dictionary name. */
abstract Property<String> getSourceName()
/** Source dictionary version or explicit unavailable marker. */
abstract Property<String> getSourceVersion()
/** Exact upstream revision or the explicit legacy-import sentinel. */
abstract Property<String> getSourceRevision()
/** Upstream source project. */
abstract Property<String> getSourceProject()
/** Official upstream repository URL. */
abstract Property<String> getSourceRepository()
/** Upstream dataset identity. */
abstract Property<String> getSourceDataset()
/** Whether the source revision is recorded or was not recorded by a legacy import. */
abstract Property<String> getSourceRevisionStatus()
/** SPDX license identifier. */
abstract Property<String> getSourceLicense()
/** Canonical URI for the source-data license. */
abstract Property<String> getSourceLicenseUri()
/** Upstream attribution supplied with the source data. */
abstract Property<String> getSourceAttribution()
/** Date on which the upstream metadata was verified. */
abstract Property<String> getSourceVerificationDate()
/** Material transformations applied by Radixor. */
abstract Property<String> getTransformationsSummary()
/** Model-specific data notice input file name, when required. */
abstract Property<String> getNoticeFileName()
/** License input file name. */
abstract Property<String> getLicenseFileName()
/** Creates the extension. */
@Inject
RadixorModelExtension(final ObjectFactory objects) {
defaultModel.convention(false)
sourceVersion.convention('unavailable')
sourceLicense.convention('LicenseRef-Radixor-Stemmer-Data')
licenseFileName.convention('LICENSE-stemmer-data.txt')
noticeFileName.convention('NOTICE-model-data.txt')
}
}

View File

@@ -0,0 +1,505 @@
package org.egothor.radixor
import org.gradle.api.GradleException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.file.DuplicatesStrategy
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.publish.PublishingExtension
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.tasks.Copy
import org.gradle.api.tasks.bundling.Jar
import org.gradle.api.tasks.bundling.Zip
import org.gradle.plugins.signing.SigningExtension
import java.nio.charset.CodingErrorAction
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.security.MessageDigest
import java.util.zip.GZIPInputStream
/** Configures validation, generation, packaging, and publication for one model artifact. */
final class RadixorModelPlugin implements Plugin<Project> {
/** Applies the model convention to a project. */
@Override
void apply(final Project project) {
project.pluginManager.apply(JavaPlugin)
project.pluginManager.apply('maven-publish')
project.pluginManager.apply('signing')
project.java {
withSourcesJar()
withJavadocJar()
sourceCompatibility = org.gradle.api.JavaVersion.VERSION_21
targetCompatibility = org.gradle.api.JavaVersion.VERSION_21
}
final RadixorModelExtension model = project.extensions.create('radixorModel', RadixorModelExtension)
project.group = 'org.egothor'
project.version = project.providers.gradleProperty('modelReleaseVersion')
.orElse(project.providers.fileContents(project.layout.projectDirectory.file('model-version.txt')).asText.map(String::trim))
.get()
final File input = project.file('src/modelInput/stemmer.gz')
final File generated = project.layout.buildDirectory.dir('generated/modelResources').get().asFile
project.sourceSets.main.resources.setSrcDirs([generated])
final def validate = project.tasks.register('validateModelInput', ValidateModelInputTask) {
group = 'verification'
description = 'Validates the immutable source dictionary, metadata, version, and model-specific licensing material.'
dictionaryFile = project.layout.projectDirectory.file('src/modelInput/stemmer.gz')
versionFile = project.layout.projectDirectory.file('model-version.txt')
modelId = model.modelId
moduleName = project.name
shareAlike = model.sourceLicense.map { String license -> license == 'CC-BY-SA-3.0' }
metadata.put('source.project', model.sourceProject)
metadata.put('source.repository', model.sourceRepository)
metadata.put('source.dataset', model.sourceDataset)
metadata.put('source.revision', model.sourceRevision)
metadata.put('source.revisionStatus', model.sourceRevisionStatus)
metadata.put('source.license', model.sourceLicense)
metadata.put('source.licenseUri', model.sourceLicenseUri)
metadata.put('source.attribution', model.sourceAttribution)
metadata.put('source.verificationDate', model.sourceVerificationDate)
metadata.put('transformations.summary', model.transformationsSummary)
}
final def prepare = project.tasks.register('prepareModelResources', PrepareModelResourcesTask) {
group = 'build'
description = 'Copies validated dictionary bytes and generates the immutable model descriptor and index.'
dependsOn(validate)
dictionaryFile = project.layout.projectDirectory.file('src/modelInput/stemmer.gz')
versionFile = project.layout.projectDirectory.file('model-version.txt')
shareAlike = model.sourceLicense.map { String license -> license == 'CC-BY-SA-3.0' }
generatedDirectory = project.layout.buildDirectory.dir('generated/modelResources')
descriptorValues.put('model.id', model.modelId)
descriptorValues.put('model.language', model.language)
descriptorValues.put('model.displayName', model.displayName)
descriptorValues.put('model.default', model.defaultModel.map(String::valueOf))
descriptorValues.put('source.name', model.sourceName)
descriptorValues.put('source.version', model.sourceVersion)
descriptorValues.put('source.project', model.sourceProject)
descriptorValues.put('source.repository', model.sourceRepository)
descriptorValues.put('source.dataset', model.sourceDataset)
descriptorValues.put('source.revision', model.sourceRevision)
descriptorValues.put('source.revisionStatus', model.sourceRevisionStatus)
descriptorValues.put('source.license', model.sourceLicense)
descriptorValues.put('source.licenseUri', model.sourceLicenseUri)
descriptorValues.put('source.attribution', model.sourceAttribution)
descriptorValues.put('source.verificationDate', model.sourceVerificationDate)
descriptorValues.put('transformations.summary', model.transformationsSummary)
}
project.afterEvaluate {
final boolean shareAlike = model.sourceLicense.get() == 'CC-BY-SA-3.0'
if (shareAlike) {
final def notice = project.layout.projectDirectory.file("src/modelInput/${model.noticeFileName.get()}")
validate.configure { noticeFile = notice }
prepare.configure { noticeFile = notice }
} else {
final def license = project.layout.projectDirectory.file("src/modelInput/${model.licenseFileName.get()}")
validate.configure { licenseFile = license }
prepare.configure { licenseFile = license }
}
}
project.tasks.named('processResources', Copy).configure { dependsOn(prepare); duplicatesStrategy = DuplicatesStrategy.FAIL }
project.tasks.named('sourcesJar', Jar).configure { dependsOn(prepare); exclude('**/stemmer.gz') }
project.tasks.named('javadocJar', Jar).configure { exclude('**/stemmer.gz') }
project.tasks.named('jar', Jar).configure {
archiveBaseName.set("radixor-model-${project.name}")
preserveFileTimestamps = false
reproducibleFileOrder = true
}
final def verifyDescriptor = project.tasks.register('verifyModelDescriptor') {
group = 'verification'; description = 'Verifies generated descriptor identity and checksum.'; dependsOn(prepare)
doLast {
final Properties properties = new Properties()
new File(generated, "META-INF/radixor/models/${model.modelId.get()}.properties").withInputStream(properties::load)
if (properties.getProperty('model.sha256') != sha256(input)) {
throw new GradleException('Generated descriptor checksum does not match the immutable source input.')
}
}
}
final def verifyJar = project.tasks.register('verifyModelJar') {
group = 'verification'; description = 'Verifies the model JAR checksum, layout, metadata, and dictionary-free documentation artifacts.'
dependsOn(project.tasks.named('jar'), project.tasks.named('sourcesJar'), project.tasks.named('javadocJar'))
doLast {
final File archive = project.tasks.named('jar', Jar).get().archiveFile.get().asFile
final List<String> names = []
final String resource = "org/egothor/stemmer/models/${model.modelId.get()}/stemmer.gz"
final boolean shareAlike = model.sourceLicense.get() == 'CC-BY-SA-3.0'
final String licenseResource = 'META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt'
final File sourceLicense = shareAlike ? null : project.file("src/modelInput/${model.licenseFileName.get()}")
final File sourceNotice = shareAlike
? project.file("src/modelInput/${model.noticeFileName.get()}") : null
final String noticeResource = "META-INF/NOTICE/${model.modelId.get()}-data.txt"
String packagedChecksum
String packagedLicenseChecksum
String packagedNoticeChecksum
new java.util.zip.ZipFile(archive).withCloseable { zip ->
zip.entries().each { names.add(it.name) }
final def entry = zip.getEntry(resource)
if (entry != null) {
packagedChecksum = sha256(zip.getInputStream(entry).bytes)
}
final def licenseEntry = zip.getEntry(licenseResource)
if (licenseEntry != null) {
packagedLicenseChecksum = sha256(zip.getInputStream(licenseEntry).bytes)
}
final def noticeEntry = zip.getEntry(noticeResource)
if (noticeEntry != null) {
packagedNoticeChecksum = sha256(zip.getInputStream(noticeEntry).bytes)
}
}
if (names.count { String name -> name.endsWith('/stemmer.gz') } != 1 || !names.contains(resource)) {
throw new GradleException("Model JAR must contain exactly one dictionary at ${resource}.")
}
if (packagedChecksum != sha256(input)) {
throw new GradleException("Packaged dictionary checksum does not match the immutable source input at ${resource}.")
}
if (shareAlike) {
requireMatchingChecksum('notice', noticeResource, sha256(sourceNotice), packagedNoticeChecksum)
validateUniMorphJarContents(names)
} else {
requireMatchingChecksum('license', licenseResource, sha256(sourceLicense), packagedLicenseChecksum)
validatePoliMorfJarContents(names)
}
['META-INF/radixor/models.index', "META-INF/radixor/models/${model.modelId.get()}.properties"].each { String name ->
if (!names.contains(name)) throw new GradleException("Model JAR is missing ${name}.")
}
[project.tasks.named('sourcesJar', Jar).get(), project.tasks.named('javadocJar', Jar).get()].each { Jar task ->
final File documentationArchive = task.archiveFile.get().asFile
new java.util.zip.ZipFile(documentationArchive).withCloseable { zip ->
if (zip.entries().any { entry -> entry.name.endsWith('/stemmer.gz') || entry.name == 'stemmer.gz' }) {
throw new GradleException("Documentation artifact ${documentationArchive.name} must not contain a model dictionary.")
}
}
}
}
}
project.tasks.register('validateModelRelease') {
group = 'verification'; description = 'Validates a tag-supplied model release version.'; dependsOn(verifyDescriptor, verifyJar)
doLast {
if (!project.hasProperty('modelReleaseVersion')) throw new GradleException('Model release validation requires -PmodelReleaseVersion=<version>.')
final String recorded = project.file('model-version.txt').text.trim()
if (project.property('modelReleaseVersion').toString() != recorded) throw new GradleException("Release version does not match model-version.txt: ${recorded}")
}
}
project.tasks.named('check').configure { dependsOn(verifyDescriptor, verifyJar) }
project.extensions.configure(PublishingExtension) { PublishingExtension publishing ->
publishing.publications.create('model', MavenPublication) { MavenPublication publication ->
publication.from(project.components.java)
publication.artifactId = "radixor-model-${project.name}"
publication.pom {
name.set("Radixor model ${project.name}")
description.set(model.displayName.zip(model.sourceLicense) { String displayName, String licenseId ->
final String material = licenseId == 'CC-BY-SA-3.0'
? 'See the packaged model-specific notice.'
: 'See the packaged model-data license.'
return "${displayName}. This artifact contains Radixor-derived model data licensed under ${licenseId}; "
.concat("Radixor software is licensed separately under BSD-3-Clause. ${material}")
})
url.set('https://github.com/leogalambos/Radixor')
licenses {
license {
name.set(model.sourceLicense)
url.set(model.sourceLicenseUri)
distribution.set('repo')
}
}
developers {
developer {
id.set('egothor')
name.set('Leo Galambos')
email.set('egothor@gmail.com')
}
}
scm {
url.set('https://github.com/leogalambos/Radixor')
connection.set('scm:git:https://github.com/leogalambos/Radixor.git')
developerConnection.set('scm:git:ssh://git@github.com/leogalambos/Radixor.git')
}
}
}
publishing.repositories.maven {
name = 'modelStaging'
url = project.layout.buildDirectory.dir('model-staging-repository').get().asFile.toURI()
}
}
final String signingKey = project.providers.environmentVariable('SIGNING_KEY').orNull
final String signingPassword = project.providers.environmentVariable('SIGNING_PASSWORD').orNull
project.extensions.configure(SigningExtension) { SigningExtension signing ->
signing.required = {
project.providers.environmentVariable('GITHUB_REF_TYPE').orNull == 'tag'
}
if (signingKey != null && !signingKey.isBlank()) {
signing.useInMemoryPgpKeys(signingKey, signingPassword)
signing.sign(project.extensions.getByType(PublishingExtension).publications.getByName('model'))
}
}
final def checksums = project.tasks.register('createModelCentralChecksums') {
group = 'publishing'
description = 'Creates Maven Central checksums for this model staging repository.'
dependsOn(project.tasks.named('publishModelPublicationToModelStagingRepository'))
doLast {
final File repository = project.layout.buildDirectory.dir('model-staging-repository').get().asFile
repository.eachFileRecurse { File artifact ->
if (artifact.isFile() && !['.md5', '.sha1', '.sha256', '.sha512'].any {
String extension -> artifact.name.endsWith(extension)
}) {
new File(artifact.absolutePath + '.md5').setText(sha256WithAlgorithm(artifact, 'MD5'), 'US-ASCII')
new File(artifact.absolutePath + '.sha1').setText(sha256WithAlgorithm(artifact, 'SHA-1'), 'US-ASCII')
}
}
}
}
project.tasks.register('packageModelReleaseCandidate', Zip) {
group = 'distribution'
description = 'Packages only this model publication as a Maven-layout local release candidate.'
dependsOn(checksums)
from(project.layout.buildDirectory.dir('model-staging-repository')) {
exclude('**/maven-metadata*.xml*')
}
destinationDirectory.set(project.layout.buildDirectory.dir('model-release-candidate'))
archiveFileName.set('central-bundle.zip')
doFirst {
if (project.providers.environmentVariable('GITHUB_REF_TYPE').orNull == 'tag'
&& (signingKey == null || signingKey.isBlank()
|| signingPassword == null || signingPassword.isBlank())) {
throw new GradleException('A tagged model release requires SIGNING_KEY and SIGNING_PASSWORD.')
}
}
}
}
/** Ensures a required file exists. */
static void requireFile(final File file, final String diagnostic) {
if (!file.isFile()) throw new GradleException(diagnostic)
}
/** Rejects a missing or byte-different packaged licensing resource. */
static void requireMatchingChecksum(final String kind, final String resource,
final String sourceChecksum, final String packagedChecksum) {
if (packagedChecksum != sourceChecksum) {
throw new GradleException("Packaged ${kind} does not match the source ${kind} at ${resource}.")
}
}
/** Validates complete source, licensing, attribution, revision-status, and transformation metadata. */
private static void validateMetadata(final RadixorModelExtension model) {
final Map<String, String> required = [
'source.project': model.sourceProject.orNull,
'source.repository': model.sourceRepository.orNull,
'source.dataset': model.sourceDataset.orNull,
'source.revision': model.sourceRevision.orNull,
'source.revisionStatus': model.sourceRevisionStatus.orNull,
'source.license': model.sourceLicense.orNull,
'source.licenseUri': model.sourceLicenseUri.orNull,
'source.attribution': model.sourceAttribution.orNull,
'source.verificationDate': model.sourceVerificationDate.orNull,
'transformations.summary': model.transformationsSummary.orNull]
required.each { String key, String value ->
if (value == null || value.isBlank()) {
throw new GradleException("Required model metadata is missing: ${key}")
}
}
validateRevisionMetadata(model.sourceRevision.get(), model.sourceRevisionStatus.get())
}
/** Accepts an exact recorded revision or the explicit legacy-import sentinel, but never an absent status. */
static void validateRevisionMetadata(final String revision, final String status) {
if (revision == null || revision.isBlank()) {
throw new GradleException('Required model metadata is missing: source.revision')
}
if (status == null || status.isBlank()) {
throw new GradleException('Required model metadata is missing: source.revisionStatus')
}
final String sentinel = 'not-recorded-in-legacy-import'
if (revision == sentinel && status != sentinel) {
throw new GradleException('The legacy revision sentinel requires source.revisionStatus=not-recorded-in-legacy-import.')
}
if (revision != sentinel && status != 'recorded') {
throw new GradleException('An exact source revision requires source.revisionStatus=recorded.')
}
}
/** Validates the model-specific attribution and ShareAlike notice. */
static void validateShareAlikeNotice(final File notice, final RadixorModelExtension model) {
validateShareAlikeNoticeText(notice.getText('UTF-8'), notice.toString(), model.modelId.get(),
model.sourceRepository.get(), model.sourceLicenseUri.get(), model.sourceRevision.get(),
model.sourceRevisionStatus.get())
}
/** Validates required content in one UniMorph model-data notice. */
static void validateShareAlikeNoticeText(final String text, final String noticeName,
final String modelId, final String repository, final String licenseUri,
final String revision, final String revisionStatus) {
final List<String> required = [
"Model ID: ${modelId}",
"Official repository: ${repository}",
'Attribution:',
'License:\nCreative Commons Attribution-ShareAlike 3.0 Unported',
"Canonical license URI: ${licenseUri}",
'Radixor modifications:',
"Revision status: ${revisionStatus}",
'Copyright (C) 2026, Leo Galambos.',
'Radixor-specific selection, verification, cleaning, normalization,',
'to the extent protected by applicable law.',
'The underlying morphological data remains attributed to UniMorph and',
"This derived model data, including Radixor's protectable contributions,",
'is distributed under Creative Commons Attribution-ShareAlike 3.0',
'Neither UniMorph nor any upstream contributor endorses Radixor.']
if (revision == 'not-recorded-in-legacy-import') {
required.add('The exact UniMorph commit used for the original Radixor import was not recorded.')
}
final List<String> missing = required.findAll { String value -> !text.contains(value) }
if (!missing.isEmpty()) {
throw new GradleException("Model notice ${noticeName} is missing required content: ${missing.join(', ')}")
}
}
/** Rejects generic license files and foreign notices in a UniMorph model artifact. */
static void validateUniMorphJarContents(final List<String> names) {
if (names.any { String name -> name.startsWith('META-INF/LICENSES/') }) {
throw new GradleException('A UniMorph model artifact must use only its model-specific notice for data licensing.')
}
if (names.count { String name -> name.startsWith('META-INF/NOTICE/') && !name.endsWith('/') } != 1) {
throw new GradleException('A UniMorph model artifact must contain exactly one model-specific notice.')
}
}
/** Rejects UniMorph licensing material in the separately licensed PoliMorf artifact. */
static void validatePoliMorfJarContents(final List<String> names) {
if (names.any { String name -> name.startsWith('META-INF/NOTICE/')
|| name.contains('CC-BY-SA') }) {
throw new GradleException('The PoliMorf artifact must not contain UniMorph CC BY-SA material.')
}
}
/** Memory-bounded validation statistics for one dictionary input. */
static final class DictionaryValidationResult {
final long acceptedGroupCount
final long acceptedFormCount
final long ignoredEmptyVariantCount
DictionaryValidationResult(final long acceptedGroupCount, final long acceptedFormCount,
final long ignoredEmptyVariantCount) {
this.acceptedGroupCount = acceptedGroupCount
this.acceptedFormCount = acceptedFormCount
this.ignoredEmptyVariantCount = ignoredEmptyVariantCount
}
}
/** Validates GZip, strict UTF-8, and dictionary rows without retaining decompressed input. */
static DictionaryValidationResult validateDictionary(final File file) {
long acceptedGroups = 0L
long acceptedForms = 0L
long ignoredEmptyVariants = 0L
try {
final def decoder = StandardCharsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
Files.newInputStream(file.toPath()).withCloseable { InputStream source ->
new BufferedInputStream(source).withCloseable { BufferedInputStream bufferedInput ->
new GZIPInputStream(bufferedInput).withCloseable { GZIPInputStream gzipInput ->
new BufferedReader(new InputStreamReader(gzipInput, decoder)).withCloseable { BufferedReader reader ->
String line
long lineNumber = 0L
while ((line = reader.readLine()) != null) {
lineNumber++
final String trimmed = line.trim()
if (trimmed && !trimmed.startsWith('#') && !trimmed.startsWith('//')) {
final String[] columns = line.split('\\t', -1)
if (columns[0].isEmpty()) {
throw new GradleException("Invalid Radixor dictionary row ${lineNumber} in ${file}.")
}
if (containsUnicodeWhitespace(columns[0])) continue
long acceptedRowForms = 1L
for (int index = 1; index < columns.length; index++) {
final String variant = columns[index]
if (variant.isEmpty()) {
ignoredEmptyVariants++
} else if (!containsUnicodeWhitespace(variant)) {
acceptedRowForms++
}
}
acceptedGroups++
acceptedForms += acceptedRowForms
}
}
}
}
}
}
} catch (GradleException exception) {
throw exception
} catch (Exception exception) {
throw new GradleException("Invalid GZip or UTF-8 model input: ${file}", exception)
}
if (acceptedGroups == 0L) throw new GradleException("Model dictionary contains no valid rows: ${file}")
if (ignoredEmptyVariants > 0L) {
println("Model validation warning: " + file + " contains " + ignoredEmptyVariants
+ " empty variant columns; the production parser intentionally ignores empty variants.")
}
return new DictionaryValidationResult(acceptedGroups, acceptedForms, ignoredEmptyVariants)
}
/** Detects Unicode whitespace in one bounded dictionary field. */
private static boolean containsUnicodeWhitespace(final String value) {
for (int index = 0; index < value.length(); index++) {
if (Character.isWhitespace(value.charAt(index))) return true
}
return false
}
/** Builds deterministic descriptor text. */
private static String descriptorText(final RadixorModelExtension model, final String version,
final String resource, final String checksum) {
return """model.id=${model.modelId.get()}
model.version=${version}
model.language=${model.language.get()}
model.displayName=${model.displayName.get()}
model.resource=${resource}
model.default=${model.defaultModel.get()}
model.format=radixor-dictionary-tsv-gzip
model.formatVersion=1
model.sha256=${checksum}
model.rightToLeft=${['FA_IR', 'HE_IL', 'YI'].contains(model.language.get())}
model.caseProcessing=LOWERCASE_WITH_LOCALE_ROOT
model.diacriticProcessing=AS_IS
model.storeOriginal=true
source.name=${model.sourceName.get()}
source.version=${model.sourceVersion.get()}
source.project=${model.sourceProject.get()}
source.repository=${model.sourceRepository.get()}
source.dataset=${model.sourceDataset.get()}
source.revision=${model.sourceRevision.get()}
source.revisionStatus=${model.sourceRevisionStatus.get()}
source.license=${model.sourceLicense.get()}
source.licenseUri=${model.sourceLicenseUri.get()}
source.attribution=${model.sourceAttribution.get()}
source.verificationDate=${model.sourceVerificationDate.get()}
transformations.summary=${model.transformationsSummary.get()}
compiler.radixorVersion=3.x
compiler.radixorCommit=unavailable
statistics.groups=unavailable
statistics.forms=unavailable
"""
}
/** Calculates the lowercase hexadecimal SHA-256 digest. */
private static String sha256(final File file) {
return sha256(file.bytes)
}
/** Calculates the lowercase hexadecimal SHA-256 digest of bytes. */
private static String sha256(final byte[] bytes) {
return MessageDigest.getInstance('SHA-256').digest(bytes).collect { byte value -> String.format('%02x', value & 0xff) }.join()
}
/** Calculates a lowercase hexadecimal digest using the requested algorithm. */
private static String sha256WithAlgorithm(final File file, final String algorithm) {
return MessageDigest.getInstance(algorithm).digest(file.bytes)
.collect { byte value -> String.format('%02x', value & 0xff) }.join()
}
}

View File

@@ -0,0 +1,57 @@
package org.egothor.radixor
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.TaskAction
/** Validates one immutable model input without retaining Project state. */
abstract class ValidateModelInputTask extends DefaultTask {
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getDictionaryFile()
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getVersionFile()
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getLicenseFile()
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getNoticeFile()
@Input abstract Property<String> getModelId()
@Input abstract Property<String> getModuleName()
@Input abstract Property<Boolean> getShareAlike()
@Input abstract MapProperty<String, String> getMetadata()
/** Performs deterministic metadata, licensing, and streaming dictionary validation. */
@TaskAction
void validateInput() {
final File dictionary = dictionaryFile.get().asFile
final String id = modelId.get()
final String version = versionFile.get().asFile.getText('UTF-8').trim()
if (id != moduleName.get() || !(id ==~ /[a-z]{2}(?:-[a-z]{2})?-[a-z0-9]+(?:-[a-z0-9]+)*/)) {
throw new GradleException("Model ID '${id}' must equal module '${moduleName.get()}' and use the safe model-ID syntax.")
}
if (!(version ==~ /[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?/)) {
throw new GradleException("Invalid semantic model version '${version}'.")
}
final Map<String, String> values = metadata.get()
values.each { String key, String value ->
if (value == null || value.isBlank()) throw new GradleException("Required model metadata is missing: ${key}")
}
RadixorModelPlugin.validateRevisionMetadata(values['source.revision'], values['source.revisionStatus'])
if (shareAlike.get()) {
final File notice = noticeFile.get().asFile
RadixorModelPlugin.validateShareAlikeNoticeText(notice.getText('UTF-8'), notice.toString(), id,
values['source.repository'], values['source.licenseUri'], values['source.revision'],
values['source.revisionStatus'])
} else {
final String text = licenseFile.get().asFile.getText('UTF-8')
if (!text.contains('SPDX-License-Identifier: BSD-2-Clause')
|| !text.contains('Copyright (c) 2016, Marcin Miłkowski')) {
throw new GradleException('The PoliMorf license must contain the complete BSD-2-Clause text and upstream attribution.')
}
}
RadixorModelPlugin.validateDictionary(dictionary)
}
}

View File

@@ -0,0 +1,205 @@
package org.egothor.radixor
import org.gradle.api.GradleException
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.util.zip.GZIPOutputStream
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertThrows
import static org.junit.jupiter.api.Assertions.assertTrue
/** Tests model licensing metadata and packaged-resource validation boundaries. */
final class RadixorModelPluginTest {
@TempDir
Path temporaryDirectory
/** Accepts a known exact source revision. */
@Test
void acceptsKnownExactRevision() {
RadixorModelPlugin.validateRevisionMetadata('6e63b53', 'recorded')
}
/** Accepts the explicit legacy-import sentinel without fabricating a revision. */
@Test
void acceptsUnknownLegacyRevision() {
RadixorModelPlugin.validateRevisionMetadata(
'not-recorded-in-legacy-import', 'not-recorded-in-legacy-import')
}
/** Rejects a missing revision-status declaration. */
@Test
void rejectsMissingRevisionStatus() {
assertThrows(GradleException) {
RadixorModelPlugin.validateRevisionMetadata('6e63b53', '')
}
}
/** Rejects a missing model-specific notice input. */
@Test
void rejectsMissingLicensingInputs() {
File missing = new File('build/nonexistent-model-licensing-input')
assertThrows(GradleException) {
RadixorModelPlugin.requireFile(missing, 'Required model notice is missing')
}
}
/** Accepts a complete model-specific UniMorph notice. */
@Test
void acceptsCompleteUniMorphNotice() {
validateNotice(validNotice())
}
/** Rejects each independently required notice statement. */
@Test
void rejectsIncompleteUniMorphNotices() {
[
'Copyright (C) 2026, Leo Galambos.',
'Attribution:',
'Creative Commons Attribution-ShareAlike 3.0 Unported',
'Canonical license URI:',
"This derived model data, including Radixor's protectable contributions,",
'Radixor modifications:',
'Revision status:',
'Neither UniMorph nor any upstream contributor endorses Radixor.'
].each { String required ->
assertThrows(GradleException) {
validateNotice(validNotice().replace(required, 'omitted'))
}
}
}
/** Rejects packaged notice bytes that differ from their model-module source. */
@Test
void rejectsIncorrectPackagedNotice() {
assertThrows(GradleException) {
RadixorModelPlugin.requireMatchingChecksum(
'notice', 'META-INF/NOTICE/test-model-data.txt', 'source', 'different')
}
}
/** Rejects UniMorph CC material in the separately licensed PoliMorf artifact. */
@Test
void rejectsUniMorphMaterialInPoliMorf() {
assertThrows(GradleException) {
RadixorModelPlugin.validatePoliMorfJarContents(
['META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt', 'META-INF/NOTICE/test-data.txt'])
}
assertThrows(GradleException) {
RadixorModelPlugin.validatePoliMorfJarContents(
['META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt', 'META-INF/LICENSES/CC-BY-SA-3.0.txt'])
}
}
/** Streams a large dictionary while retaining only aggregate counters and the current row. */
@Test
void validatesLargeDictionaryWithBoundedState() {
final int groups = 250_000
final File dictionary = temporaryDirectory.resolve('large.gz').toFile()
writeGzip(dictionary) { BufferedWriter writer ->
for (int index = 0; index < groups; index++) {
writer.write("stem${index}\tvariant${index}\t\n")
}
}
final RadixorModelPlugin.DictionaryValidationResult result =
RadixorModelPlugin.validateDictionary(dictionary)
assertEquals(groups, result.acceptedGroupCount)
assertEquals(groups * 2L, result.acceptedFormCount)
assertEquals(groups, result.ignoredEmptyVariantCount)
}
/** Rejects a source that is not a GZip stream. */
@Test
void rejectsInvalidGzip() {
final File dictionary = temporaryDirectory.resolve('invalid.gz').toFile()
Files.writeString(dictionary.toPath(), 'not gzip', StandardCharsets.UTF_8)
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
}
/** Rejects malformed UTF-8 through the strict incremental decoder. */
@Test
void rejectsMalformedUtf8() {
final File dictionary = temporaryDirectory.resolve('malformed-utf8.gz').toFile()
new GZIPOutputStream(Files.newOutputStream(dictionary.toPath())).withCloseable { OutputStream output ->
output.write([0x73, 0x74, 0x65, 0x6d, 0x09, 0xc3, 0x28, 0x0a] as byte[])
}
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
}
/** Rejects structurally invalid rows with an empty stem. */
@Test
void rejectsInvalidRows() {
final File dictionary = temporaryDirectory.resolve('invalid-row.gz').toFile()
writeGzip(dictionary) { BufferedWriter writer -> writer.write("\tvariant\n") }
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
}
/** Preserves the production parser policy for Unicode-whitespace items. */
@Test
void rejectsUnicodeWhitespaceItemsWithoutRejectingTheSource() {
final File dictionary = temporaryDirectory.resolve('whitespace-items.gz').toFile()
writeGzip(dictionary) { BufferedWriter writer ->
writer.write("invalid stem\tvariant\n")
writer.write("valid\taccepted\tinvalid variant\n")
}
final RadixorModelPlugin.DictionaryValidationResult result =
RadixorModelPlugin.validateDictionary(dictionary)
assertEquals(1L, result.acceptedGroupCount)
assertEquals(2L, result.acceptedFormCount)
}
/** Streams the complete maintained PoliMorf model input successfully. */
@Test
void validatesFullPoliMorfInput() {
final List<File> candidates = [
new File('models/pl-pl-polimorf/src/modelInput/stemmer.gz'),
new File('../models/pl-pl-polimorf/src/modelInput/stemmer.gz')]
final File dictionary = candidates.find { File candidate -> candidate.isFile() }
assertTrue(dictionary != null, 'The complete PoliMorf model input must be available to build-logic tests.')
final RadixorModelPlugin.DictionaryValidationResult result =
RadixorModelPlugin.validateDictionary(dictionary)
assertTrue(result.acceptedGroupCount > 0L)
assertTrue(result.acceptedFormCount > result.acceptedGroupCount)
}
private static void writeGzip(final File target, final Closure<Void> content) {
new GZIPOutputStream(Files.newOutputStream(target.toPath())).withCloseable { OutputStream gzip ->
new BufferedWriter(new OutputStreamWriter(gzip, StandardCharsets.UTF_8)).withCloseable {
BufferedWriter writer -> content.call(writer)
}
}
}
private static void validateNotice(final String text) {
RadixorModelPlugin.validateShareAlikeNoticeText(text, 'test notice', 'test-model',
'https://github.com/unimorph/test', 'https://creativecommons.org/licenses/by-sa/3.0/',
'not-recorded-in-legacy-import', 'not-recorded-in-legacy-import')
}
private static String validNotice() {
return '''Model ID: test-model
Official repository: https://github.com/unimorph/test
Attribution: UniMorph and upstream contributors
License:
Creative Commons Attribution-ShareAlike 3.0 Unported
Canonical license URI: https://creativecommons.org/licenses/by-sa/3.0/
Radixor modifications: Cleaning and packaging.
Revision status: not-recorded-in-legacy-import
The exact UniMorph commit used for the original Radixor import was not recorded.
Copyright (C) 2026, Leo Galambos.
Radixor-specific selection, verification, cleaning, normalization,
to the extent protected by applicable law.
The underlying morphological data remains attributed to UniMorph and
This derived model data, including Radixor's protectable contributions,
is distributed under Creative Commons Attribution-ShareAlike 3.0
Neither UniMorph nor any upstream contributor endorses Radixor.
'''
}
}