feat!: modularize stemmer models and release infrastructure
Move bundled stemmer dictionaries from the core artifact into independently versioned model modules. Add model discovery and explicit model-loading APIs, a standard model aggregate, a model BOM, and dedicated model and catalog release workflows. Add full PoliMorf integration, model provenance and licensing validation, streaming model-input verification, strict dependency verification, consumer resolution tests, Configuration Cache compatibility, and expanded JMH, quality, documentation, and release checks. Upgrade the CycloneDX and JMH Gradle plugins and remove Gradle 10 and Java compiler deprecations. BREAKING CHANGE: The core Radixor artifact no longer contains bundled stemmer dictionaries. Applications must add the required model artifacts, the standard model aggregate, or model dependencies managed through the Radixor model BOM.
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.file.ConfigurableFileCollection
|
||||
import org.gradle.api.tasks.Classpath
|
||||
import org.gradle.process.CommandLineArgumentProvider
|
||||
|
||||
import javax.inject.Inject
|
||||
|
||||
abstract class MockitoAgentArgumentProvider implements CommandLineArgumentProvider {
|
||||
@Classpath
|
||||
abstract ConfigurableFileCollection getAgentClasspath()
|
||||
|
||||
@Inject
|
||||
MockitoAgentArgumentProvider() {
|
||||
}
|
||||
|
||||
@Override
|
||||
Iterable<String> asArguments() {
|
||||
return ["-javaagent:${agentClasspath.singleFile.absolutePath}"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.GradleException
|
||||
import org.gradle.api.file.ConfigurableFileCollection
|
||||
import org.gradle.api.file.DirectoryProperty
|
||||
import org.gradle.api.file.RegularFileProperty
|
||||
import org.gradle.api.provider.MapProperty
|
||||
import org.gradle.api.provider.Property
|
||||
import org.gradle.api.tasks.Input
|
||||
import org.gradle.api.tasks.InputFile
|
||||
import org.gradle.api.tasks.InputFiles
|
||||
import org.gradle.api.tasks.OutputDirectory
|
||||
import org.gradle.api.tasks.PathSensitive
|
||||
import org.gradle.api.tasks.PathSensitivity
|
||||
import org.gradle.api.tasks.TaskAction
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.util.stream.Stream
|
||||
|
||||
/** Builds the isolated Maven-layout repository used by consumer resolution tests. */
|
||||
abstract class PrepareModelConsumerRepositoryTask extends DefaultTask {
|
||||
@Input abstract Property<String> getCoreVersion()
|
||||
@Input abstract Property<String> getCatalogVersion()
|
||||
@Input abstract MapProperty<String, String> getModelVersions()
|
||||
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract RegularFileProperty getCorePom()
|
||||
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract RegularFileProperty getCoreJar()
|
||||
|
||||
@InputFiles @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract ConfigurableFileCollection getModelPoms()
|
||||
|
||||
@InputFiles @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract ConfigurableFileCollection getModelJars()
|
||||
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract RegularFileProperty getStandardPom()
|
||||
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract RegularFileProperty getBomPom()
|
||||
|
||||
@OutputDirectory
|
||||
abstract DirectoryProperty getRepositoryDirectory()
|
||||
|
||||
/** Creates the repository using only declared task state and Java file APIs. */
|
||||
@TaskAction
|
||||
void prepareRepository() {
|
||||
final Path repository = repositoryDirectory.get().asFile.toPath()
|
||||
deleteTree(repository)
|
||||
Files.createDirectories(repository)
|
||||
install(repository, 'radixor', coreVersion.get(), corePom.get().asFile.toPath(), coreJar.get().asFile.toPath())
|
||||
|
||||
final Map<String, Path> pomsByModel = indexModelFiles(modelPoms.files)
|
||||
final Map<String, Path> jarsByModel = indexModelFiles(modelJars.files)
|
||||
modelVersions.get().toSorted().each { String modelId, String modelVersion ->
|
||||
final Path pom = pomsByModel.get(modelId)
|
||||
final Path jar = jarsByModel.get(modelId)
|
||||
if (pom == null || jar == null) {
|
||||
throw new GradleException("Missing generated publication input for model ${modelId}.")
|
||||
}
|
||||
PrepareModelConsumerRepositoryTask.install(
|
||||
repository, "radixor-model-${modelId}", modelVersion, pom, jar)
|
||||
}
|
||||
install(repository, 'radixor-models-standard', catalogVersion.get(), standardPom.get().asFile.toPath(), null)
|
||||
install(repository, 'radixor-models-bom', catalogVersion.get(), bomPom.get().asFile.toPath(), null)
|
||||
}
|
||||
|
||||
private static Map<String, Path> indexModelFiles(final Set<File> files) {
|
||||
final Map<String, Path> indexed = [:]
|
||||
files.each { File file ->
|
||||
Path cursor = file.toPath().toAbsolutePath().parent
|
||||
while (cursor != null && cursor.fileName.toString() != 'build') cursor = cursor.parent
|
||||
if (cursor == null || cursor.parent == null) {
|
||||
throw new GradleException("Cannot determine model ID from generated input ${file}.")
|
||||
}
|
||||
final String modelId = cursor.parent.fileName.toString()
|
||||
if (indexed.put(modelId, file.toPath()) != null) {
|
||||
throw new GradleException("Duplicate generated publication input for model ${modelId}.")
|
||||
}
|
||||
}
|
||||
return indexed
|
||||
}
|
||||
|
||||
private static void install(final Path repository, final String artifactId, final String version,
|
||||
final Path pom, final Path jar) {
|
||||
final Path module = repository.resolve("org/egothor/${artifactId}/${version}")
|
||||
Files.createDirectories(module)
|
||||
Files.copy(pom, module.resolve("${artifactId}-${version}.pom"), StandardCopyOption.REPLACE_EXISTING)
|
||||
if (jar != null) {
|
||||
Files.copy(jar, module.resolve("${artifactId}-${version}.jar"), StandardCopyOption.REPLACE_EXISTING)
|
||||
}
|
||||
}
|
||||
|
||||
private static void deleteTree(final Path directory) {
|
||||
if (!Files.exists(directory)) return
|
||||
Files.walk(directory).withCloseable { Stream<Path> paths ->
|
||||
paths.sorted(Comparator.reverseOrder()).forEach(Files::delete)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.file.DirectoryProperty
|
||||
import org.gradle.api.file.RegularFileProperty
|
||||
import org.gradle.api.provider.MapProperty
|
||||
import org.gradle.api.provider.Property
|
||||
import org.gradle.api.tasks.Input
|
||||
import org.gradle.api.tasks.InputFile
|
||||
import org.gradle.api.tasks.Optional
|
||||
import org.gradle.api.tasks.OutputDirectory
|
||||
import org.gradle.api.tasks.PathSensitive
|
||||
import org.gradle.api.tasks.PathSensitivity
|
||||
import org.gradle.api.tasks.TaskAction
|
||||
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.security.MessageDigest
|
||||
import java.util.stream.Stream
|
||||
|
||||
/** Generates one model's deterministic resource tree without retaining Project state. */
|
||||
abstract class PrepareModelResourcesTask extends DefaultTask {
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getDictionaryFile()
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getVersionFile()
|
||||
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getLicenseFile()
|
||||
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getNoticeFile()
|
||||
@Input abstract Property<Boolean> getShareAlike()
|
||||
@Input abstract MapProperty<String, String> getDescriptorValues()
|
||||
@OutputDirectory abstract DirectoryProperty getGeneratedDirectory()
|
||||
|
||||
/** Copies bounded inputs and writes descriptor and index files. */
|
||||
@TaskAction
|
||||
void prepareResources() {
|
||||
final Path generated = generatedDirectory.get().asFile.toPath()
|
||||
deleteTree(generated)
|
||||
final Map<String, String> values = descriptorValues.get()
|
||||
final String id = values['model.id']
|
||||
final String resource = "org/egothor/stemmer/models/${id}/stemmer.gz"
|
||||
final Path dictionaryTarget = generated.resolve(resource)
|
||||
Files.createDirectories(dictionaryTarget.parent)
|
||||
Files.copy(dictionaryFile.get().asFile.toPath(), dictionaryTarget, StandardCopyOption.REPLACE_EXISTING)
|
||||
|
||||
final Path descriptor = generated.resolve("META-INF/radixor/models/${id}.properties")
|
||||
Files.createDirectories(descriptor.parent)
|
||||
Files.writeString(descriptor, descriptorText(values,
|
||||
versionFile.get().asFile.getText('UTF-8').trim(), resource, sha256(dictionaryFile.get().asFile)))
|
||||
final Path index = generated.resolve('META-INF/radixor/models.index')
|
||||
Files.createDirectories(index.parent)
|
||||
Files.writeString(index, "META-INF/radixor/models/${id}.properties\n")
|
||||
|
||||
if (shareAlike.get()) {
|
||||
final Path notice = generated.resolve("META-INF/NOTICE/${id}-data.txt")
|
||||
Files.createDirectories(notice.parent)
|
||||
Files.copy(noticeFile.get().asFile.toPath(), notice, StandardCopyOption.REPLACE_EXISTING)
|
||||
} else {
|
||||
final Path license = generated.resolve('META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt')
|
||||
Files.createDirectories(license.parent)
|
||||
Files.copy(licenseFile.get().asFile.toPath(), license, StandardCopyOption.REPLACE_EXISTING)
|
||||
}
|
||||
}
|
||||
|
||||
private static String descriptorText(final Map<String, String> value, final String version,
|
||||
final String resource, final String checksum) {
|
||||
return """model.id=${value['model.id']}
|
||||
model.version=${version}
|
||||
model.language=${value['model.language']}
|
||||
model.displayName=${value['model.displayName']}
|
||||
model.resource=${resource}
|
||||
model.default=${value['model.default']}
|
||||
model.format=radixor-dictionary-tsv-gzip
|
||||
model.formatVersion=1
|
||||
model.sha256=${checksum}
|
||||
model.rightToLeft=${['FA_IR', 'HE_IL', 'YI'].contains(value['model.language'])}
|
||||
model.caseProcessing=LOWERCASE_WITH_LOCALE_ROOT
|
||||
model.diacriticProcessing=AS_IS
|
||||
model.storeOriginal=true
|
||||
source.name=${value['source.name']}
|
||||
source.version=${value['source.version']}
|
||||
source.project=${value['source.project']}
|
||||
source.repository=${value['source.repository']}
|
||||
source.dataset=${value['source.dataset']}
|
||||
source.revision=${value['source.revision']}
|
||||
source.revisionStatus=${value['source.revisionStatus']}
|
||||
source.license=${value['source.license']}
|
||||
source.licenseUri=${value['source.licenseUri']}
|
||||
source.attribution=${value['source.attribution']}
|
||||
source.verificationDate=${value['source.verificationDate']}
|
||||
transformations.summary=${value['transformations.summary']}
|
||||
compiler.radixorVersion=3.x
|
||||
compiler.radixorCommit=unavailable
|
||||
statistics.groups=unavailable
|
||||
statistics.forms=unavailable
|
||||
"""
|
||||
}
|
||||
|
||||
private static String sha256(final File file) {
|
||||
return MessageDigest.getInstance('SHA-256').digest(file.bytes)
|
||||
.collect { byte value -> String.format('%02x', value & 0xff) }.join()
|
||||
}
|
||||
|
||||
private static void deleteTree(final Path directory) {
|
||||
if (!Files.exists(directory)) return
|
||||
Files.walk(directory).withCloseable { Stream<Path> paths ->
|
||||
paths.sorted(Comparator.reverseOrder()).forEach(Files::delete)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.Plugin
|
||||
import org.gradle.api.Project
|
||||
|
||||
/** Exposes typed repository build-support tasks to the root build. */
|
||||
final class RadixorBuildSupportPlugin implements Plugin<Project> {
|
||||
/** Registers build-support tasks without inspecting project state during execution. */
|
||||
@Override
|
||||
void apply(final Project project) {
|
||||
project.tasks.register('prepareModelConsumerTestRepository', PrepareModelConsumerRepositoryTask) {
|
||||
group = 'verification'
|
||||
description = 'Creates an isolated local Maven repository for model dependency-resolution integration tests.'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.model.ObjectFactory
|
||||
import org.gradle.api.provider.Property
|
||||
|
||||
import javax.inject.Inject
|
||||
|
||||
/** Declarative configuration for one independently published Radixor model. */
|
||||
abstract class RadixorModelExtension {
|
||||
/** Stable model identifier. */
|
||||
abstract Property<String> getModelId()
|
||||
|
||||
/** Radixor language enum constant. */
|
||||
abstract Property<String> getLanguage()
|
||||
|
||||
/** Human-readable model name. */
|
||||
abstract Property<String> getDisplayName()
|
||||
|
||||
/** Whether this is the documented default for its language. */
|
||||
abstract Property<Boolean> getDefaultModel()
|
||||
|
||||
/** Source dictionary name. */
|
||||
abstract Property<String> getSourceName()
|
||||
|
||||
/** Source dictionary version or explicit unavailable marker. */
|
||||
abstract Property<String> getSourceVersion()
|
||||
|
||||
/** Exact upstream revision or the explicit legacy-import sentinel. */
|
||||
abstract Property<String> getSourceRevision()
|
||||
|
||||
/** Upstream source project. */
|
||||
abstract Property<String> getSourceProject()
|
||||
|
||||
/** Official upstream repository URL. */
|
||||
abstract Property<String> getSourceRepository()
|
||||
|
||||
/** Upstream dataset identity. */
|
||||
abstract Property<String> getSourceDataset()
|
||||
|
||||
/** Whether the source revision is recorded or was not recorded by a legacy import. */
|
||||
abstract Property<String> getSourceRevisionStatus()
|
||||
|
||||
/** SPDX license identifier. */
|
||||
abstract Property<String> getSourceLicense()
|
||||
|
||||
/** Canonical URI for the source-data license. */
|
||||
abstract Property<String> getSourceLicenseUri()
|
||||
|
||||
/** Upstream attribution supplied with the source data. */
|
||||
abstract Property<String> getSourceAttribution()
|
||||
|
||||
/** Date on which the upstream metadata was verified. */
|
||||
abstract Property<String> getSourceVerificationDate()
|
||||
|
||||
/** Material transformations applied by Radixor. */
|
||||
abstract Property<String> getTransformationsSummary()
|
||||
|
||||
/** Model-specific data notice input file name, when required. */
|
||||
abstract Property<String> getNoticeFileName()
|
||||
|
||||
/** License input file name. */
|
||||
abstract Property<String> getLicenseFileName()
|
||||
|
||||
/** Creates the extension. */
|
||||
@Inject
|
||||
RadixorModelExtension(final ObjectFactory objects) {
|
||||
defaultModel.convention(false)
|
||||
sourceVersion.convention('unavailable')
|
||||
sourceLicense.convention('LicenseRef-Radixor-Stemmer-Data')
|
||||
licenseFileName.convention('LICENSE-stemmer-data.txt')
|
||||
noticeFileName.convention('NOTICE-model-data.txt')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.GradleException
|
||||
import org.gradle.api.Plugin
|
||||
import org.gradle.api.Project
|
||||
import org.gradle.api.file.DuplicatesStrategy
|
||||
import org.gradle.api.plugins.JavaPlugin
|
||||
import org.gradle.api.publish.PublishingExtension
|
||||
import org.gradle.api.publish.maven.MavenPublication
|
||||
import org.gradle.api.tasks.Copy
|
||||
import org.gradle.api.tasks.bundling.Jar
|
||||
import org.gradle.api.tasks.bundling.Zip
|
||||
import org.gradle.plugins.signing.SigningExtension
|
||||
|
||||
import java.nio.charset.CodingErrorAction
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.Files
|
||||
import java.security.MessageDigest
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
/** Configures validation, generation, packaging, and publication for one model artifact. */
|
||||
final class RadixorModelPlugin implements Plugin<Project> {
|
||||
/** Applies the model convention to a project. */
|
||||
@Override
|
||||
void apply(final Project project) {
|
||||
project.pluginManager.apply(JavaPlugin)
|
||||
project.pluginManager.apply('maven-publish')
|
||||
project.pluginManager.apply('signing')
|
||||
project.java {
|
||||
withSourcesJar()
|
||||
withJavadocJar()
|
||||
sourceCompatibility = org.gradle.api.JavaVersion.VERSION_21
|
||||
targetCompatibility = org.gradle.api.JavaVersion.VERSION_21
|
||||
}
|
||||
final RadixorModelExtension model = project.extensions.create('radixorModel', RadixorModelExtension)
|
||||
project.group = 'org.egothor'
|
||||
project.version = project.providers.gradleProperty('modelReleaseVersion')
|
||||
.orElse(project.providers.fileContents(project.layout.projectDirectory.file('model-version.txt')).asText.map(String::trim))
|
||||
.get()
|
||||
|
||||
final File input = project.file('src/modelInput/stemmer.gz')
|
||||
final File generated = project.layout.buildDirectory.dir('generated/modelResources').get().asFile
|
||||
project.sourceSets.main.resources.setSrcDirs([generated])
|
||||
|
||||
final def validate = project.tasks.register('validateModelInput', ValidateModelInputTask) {
|
||||
group = 'verification'
|
||||
description = 'Validates the immutable source dictionary, metadata, version, and model-specific licensing material.'
|
||||
dictionaryFile = project.layout.projectDirectory.file('src/modelInput/stemmer.gz')
|
||||
versionFile = project.layout.projectDirectory.file('model-version.txt')
|
||||
modelId = model.modelId
|
||||
moduleName = project.name
|
||||
shareAlike = model.sourceLicense.map { String license -> license == 'CC-BY-SA-3.0' }
|
||||
metadata.put('source.project', model.sourceProject)
|
||||
metadata.put('source.repository', model.sourceRepository)
|
||||
metadata.put('source.dataset', model.sourceDataset)
|
||||
metadata.put('source.revision', model.sourceRevision)
|
||||
metadata.put('source.revisionStatus', model.sourceRevisionStatus)
|
||||
metadata.put('source.license', model.sourceLicense)
|
||||
metadata.put('source.licenseUri', model.sourceLicenseUri)
|
||||
metadata.put('source.attribution', model.sourceAttribution)
|
||||
metadata.put('source.verificationDate', model.sourceVerificationDate)
|
||||
metadata.put('transformations.summary', model.transformationsSummary)
|
||||
}
|
||||
|
||||
final def prepare = project.tasks.register('prepareModelResources', PrepareModelResourcesTask) {
|
||||
group = 'build'
|
||||
description = 'Copies validated dictionary bytes and generates the immutable model descriptor and index.'
|
||||
dependsOn(validate)
|
||||
dictionaryFile = project.layout.projectDirectory.file('src/modelInput/stemmer.gz')
|
||||
versionFile = project.layout.projectDirectory.file('model-version.txt')
|
||||
shareAlike = model.sourceLicense.map { String license -> license == 'CC-BY-SA-3.0' }
|
||||
generatedDirectory = project.layout.buildDirectory.dir('generated/modelResources')
|
||||
descriptorValues.put('model.id', model.modelId)
|
||||
descriptorValues.put('model.language', model.language)
|
||||
descriptorValues.put('model.displayName', model.displayName)
|
||||
descriptorValues.put('model.default', model.defaultModel.map(String::valueOf))
|
||||
descriptorValues.put('source.name', model.sourceName)
|
||||
descriptorValues.put('source.version', model.sourceVersion)
|
||||
descriptorValues.put('source.project', model.sourceProject)
|
||||
descriptorValues.put('source.repository', model.sourceRepository)
|
||||
descriptorValues.put('source.dataset', model.sourceDataset)
|
||||
descriptorValues.put('source.revision', model.sourceRevision)
|
||||
descriptorValues.put('source.revisionStatus', model.sourceRevisionStatus)
|
||||
descriptorValues.put('source.license', model.sourceLicense)
|
||||
descriptorValues.put('source.licenseUri', model.sourceLicenseUri)
|
||||
descriptorValues.put('source.attribution', model.sourceAttribution)
|
||||
descriptorValues.put('source.verificationDate', model.sourceVerificationDate)
|
||||
descriptorValues.put('transformations.summary', model.transformationsSummary)
|
||||
}
|
||||
project.afterEvaluate {
|
||||
final boolean shareAlike = model.sourceLicense.get() == 'CC-BY-SA-3.0'
|
||||
if (shareAlike) {
|
||||
final def notice = project.layout.projectDirectory.file("src/modelInput/${model.noticeFileName.get()}")
|
||||
validate.configure { noticeFile = notice }
|
||||
prepare.configure { noticeFile = notice }
|
||||
} else {
|
||||
final def license = project.layout.projectDirectory.file("src/modelInput/${model.licenseFileName.get()}")
|
||||
validate.configure { licenseFile = license }
|
||||
prepare.configure { licenseFile = license }
|
||||
}
|
||||
}
|
||||
project.tasks.named('processResources', Copy).configure { dependsOn(prepare); duplicatesStrategy = DuplicatesStrategy.FAIL }
|
||||
project.tasks.named('sourcesJar', Jar).configure { dependsOn(prepare); exclude('**/stemmer.gz') }
|
||||
project.tasks.named('javadocJar', Jar).configure { exclude('**/stemmer.gz') }
|
||||
project.tasks.named('jar', Jar).configure {
|
||||
archiveBaseName.set("radixor-model-${project.name}")
|
||||
preserveFileTimestamps = false
|
||||
reproducibleFileOrder = true
|
||||
}
|
||||
final def verifyDescriptor = project.tasks.register('verifyModelDescriptor') {
|
||||
group = 'verification'; description = 'Verifies generated descriptor identity and checksum.'; dependsOn(prepare)
|
||||
doLast {
|
||||
final Properties properties = new Properties()
|
||||
new File(generated, "META-INF/radixor/models/${model.modelId.get()}.properties").withInputStream(properties::load)
|
||||
if (properties.getProperty('model.sha256') != sha256(input)) {
|
||||
throw new GradleException('Generated descriptor checksum does not match the immutable source input.')
|
||||
}
|
||||
}
|
||||
}
|
||||
final def verifyJar = project.tasks.register('verifyModelJar') {
|
||||
group = 'verification'; description = 'Verifies the model JAR checksum, layout, metadata, and dictionary-free documentation artifacts.'
|
||||
dependsOn(project.tasks.named('jar'), project.tasks.named('sourcesJar'), project.tasks.named('javadocJar'))
|
||||
doLast {
|
||||
final File archive = project.tasks.named('jar', Jar).get().archiveFile.get().asFile
|
||||
final List<String> names = []
|
||||
final String resource = "org/egothor/stemmer/models/${model.modelId.get()}/stemmer.gz"
|
||||
final boolean shareAlike = model.sourceLicense.get() == 'CC-BY-SA-3.0'
|
||||
final String licenseResource = 'META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt'
|
||||
final File sourceLicense = shareAlike ? null : project.file("src/modelInput/${model.licenseFileName.get()}")
|
||||
final File sourceNotice = shareAlike
|
||||
? project.file("src/modelInput/${model.noticeFileName.get()}") : null
|
||||
final String noticeResource = "META-INF/NOTICE/${model.modelId.get()}-data.txt"
|
||||
String packagedChecksum
|
||||
String packagedLicenseChecksum
|
||||
String packagedNoticeChecksum
|
||||
new java.util.zip.ZipFile(archive).withCloseable { zip ->
|
||||
zip.entries().each { names.add(it.name) }
|
||||
final def entry = zip.getEntry(resource)
|
||||
if (entry != null) {
|
||||
packagedChecksum = sha256(zip.getInputStream(entry).bytes)
|
||||
}
|
||||
final def licenseEntry = zip.getEntry(licenseResource)
|
||||
if (licenseEntry != null) {
|
||||
packagedLicenseChecksum = sha256(zip.getInputStream(licenseEntry).bytes)
|
||||
}
|
||||
final def noticeEntry = zip.getEntry(noticeResource)
|
||||
if (noticeEntry != null) {
|
||||
packagedNoticeChecksum = sha256(zip.getInputStream(noticeEntry).bytes)
|
||||
}
|
||||
}
|
||||
if (names.count { String name -> name.endsWith('/stemmer.gz') } != 1 || !names.contains(resource)) {
|
||||
throw new GradleException("Model JAR must contain exactly one dictionary at ${resource}.")
|
||||
}
|
||||
if (packagedChecksum != sha256(input)) {
|
||||
throw new GradleException("Packaged dictionary checksum does not match the immutable source input at ${resource}.")
|
||||
}
|
||||
if (shareAlike) {
|
||||
requireMatchingChecksum('notice', noticeResource, sha256(sourceNotice), packagedNoticeChecksum)
|
||||
validateUniMorphJarContents(names)
|
||||
} else {
|
||||
requireMatchingChecksum('license', licenseResource, sha256(sourceLicense), packagedLicenseChecksum)
|
||||
validatePoliMorfJarContents(names)
|
||||
}
|
||||
['META-INF/radixor/models.index', "META-INF/radixor/models/${model.modelId.get()}.properties"].each { String name ->
|
||||
if (!names.contains(name)) throw new GradleException("Model JAR is missing ${name}.")
|
||||
}
|
||||
[project.tasks.named('sourcesJar', Jar).get(), project.tasks.named('javadocJar', Jar).get()].each { Jar task ->
|
||||
final File documentationArchive = task.archiveFile.get().asFile
|
||||
new java.util.zip.ZipFile(documentationArchive).withCloseable { zip ->
|
||||
if (zip.entries().any { entry -> entry.name.endsWith('/stemmer.gz') || entry.name == 'stemmer.gz' }) {
|
||||
throw new GradleException("Documentation artifact ${documentationArchive.name} must not contain a model dictionary.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
project.tasks.register('validateModelRelease') {
|
||||
group = 'verification'; description = 'Validates a tag-supplied model release version.'; dependsOn(verifyDescriptor, verifyJar)
|
||||
doLast {
|
||||
if (!project.hasProperty('modelReleaseVersion')) throw new GradleException('Model release validation requires -PmodelReleaseVersion=<version>.')
|
||||
final String recorded = project.file('model-version.txt').text.trim()
|
||||
if (project.property('modelReleaseVersion').toString() != recorded) throw new GradleException("Release version does not match model-version.txt: ${recorded}")
|
||||
}
|
||||
}
|
||||
project.tasks.named('check').configure { dependsOn(verifyDescriptor, verifyJar) }
|
||||
project.extensions.configure(PublishingExtension) { PublishingExtension publishing ->
|
||||
publishing.publications.create('model', MavenPublication) { MavenPublication publication ->
|
||||
publication.from(project.components.java)
|
||||
publication.artifactId = "radixor-model-${project.name}"
|
||||
publication.pom {
|
||||
name.set("Radixor model ${project.name}")
|
||||
description.set(model.displayName.zip(model.sourceLicense) { String displayName, String licenseId ->
|
||||
final String material = licenseId == 'CC-BY-SA-3.0'
|
||||
? 'See the packaged model-specific notice.'
|
||||
: 'See the packaged model-data license.'
|
||||
return "${displayName}. This artifact contains Radixor-derived model data licensed under ${licenseId}; "
|
||||
.concat("Radixor software is licensed separately under BSD-3-Clause. ${material}")
|
||||
})
|
||||
url.set('https://github.com/leogalambos/Radixor')
|
||||
licenses {
|
||||
license {
|
||||
name.set(model.sourceLicense)
|
||||
url.set(model.sourceLicenseUri)
|
||||
distribution.set('repo')
|
||||
}
|
||||
}
|
||||
developers {
|
||||
developer {
|
||||
id.set('egothor')
|
||||
name.set('Leo Galambos')
|
||||
email.set('egothor@gmail.com')
|
||||
}
|
||||
}
|
||||
scm {
|
||||
url.set('https://github.com/leogalambos/Radixor')
|
||||
connection.set('scm:git:https://github.com/leogalambos/Radixor.git')
|
||||
developerConnection.set('scm:git:ssh://git@github.com/leogalambos/Radixor.git')
|
||||
}
|
||||
}
|
||||
}
|
||||
publishing.repositories.maven {
|
||||
name = 'modelStaging'
|
||||
url = project.layout.buildDirectory.dir('model-staging-repository').get().asFile.toURI()
|
||||
}
|
||||
}
|
||||
|
||||
final String signingKey = project.providers.environmentVariable('SIGNING_KEY').orNull
|
||||
final String signingPassword = project.providers.environmentVariable('SIGNING_PASSWORD').orNull
|
||||
project.extensions.configure(SigningExtension) { SigningExtension signing ->
|
||||
signing.required = {
|
||||
project.providers.environmentVariable('GITHUB_REF_TYPE').orNull == 'tag'
|
||||
}
|
||||
if (signingKey != null && !signingKey.isBlank()) {
|
||||
signing.useInMemoryPgpKeys(signingKey, signingPassword)
|
||||
signing.sign(project.extensions.getByType(PublishingExtension).publications.getByName('model'))
|
||||
}
|
||||
}
|
||||
|
||||
final def checksums = project.tasks.register('createModelCentralChecksums') {
|
||||
group = 'publishing'
|
||||
description = 'Creates Maven Central checksums for this model staging repository.'
|
||||
dependsOn(project.tasks.named('publishModelPublicationToModelStagingRepository'))
|
||||
doLast {
|
||||
final File repository = project.layout.buildDirectory.dir('model-staging-repository').get().asFile
|
||||
repository.eachFileRecurse { File artifact ->
|
||||
if (artifact.isFile() && !['.md5', '.sha1', '.sha256', '.sha512'].any {
|
||||
String extension -> artifact.name.endsWith(extension)
|
||||
}) {
|
||||
new File(artifact.absolutePath + '.md5').setText(sha256WithAlgorithm(artifact, 'MD5'), 'US-ASCII')
|
||||
new File(artifact.absolutePath + '.sha1').setText(sha256WithAlgorithm(artifact, 'SHA-1'), 'US-ASCII')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
project.tasks.register('packageModelReleaseCandidate', Zip) {
|
||||
group = 'distribution'
|
||||
description = 'Packages only this model publication as a Maven-layout local release candidate.'
|
||||
dependsOn(checksums)
|
||||
from(project.layout.buildDirectory.dir('model-staging-repository')) {
|
||||
exclude('**/maven-metadata*.xml*')
|
||||
}
|
||||
destinationDirectory.set(project.layout.buildDirectory.dir('model-release-candidate'))
|
||||
archiveFileName.set('central-bundle.zip')
|
||||
doFirst {
|
||||
if (project.providers.environmentVariable('GITHUB_REF_TYPE').orNull == 'tag'
|
||||
&& (signingKey == null || signingKey.isBlank()
|
||||
|| signingPassword == null || signingPassword.isBlank())) {
|
||||
throw new GradleException('A tagged model release requires SIGNING_KEY and SIGNING_PASSWORD.')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensures a required file exists. */
|
||||
static void requireFile(final File file, final String diagnostic) {
|
||||
if (!file.isFile()) throw new GradleException(diagnostic)
|
||||
}
|
||||
|
||||
/** Rejects a missing or byte-different packaged licensing resource. */
|
||||
static void requireMatchingChecksum(final String kind, final String resource,
|
||||
final String sourceChecksum, final String packagedChecksum) {
|
||||
if (packagedChecksum != sourceChecksum) {
|
||||
throw new GradleException("Packaged ${kind} does not match the source ${kind} at ${resource}.")
|
||||
}
|
||||
}
|
||||
|
||||
/** Validates complete source, licensing, attribution, revision-status, and transformation metadata. */
|
||||
private static void validateMetadata(final RadixorModelExtension model) {
|
||||
final Map<String, String> required = [
|
||||
'source.project': model.sourceProject.orNull,
|
||||
'source.repository': model.sourceRepository.orNull,
|
||||
'source.dataset': model.sourceDataset.orNull,
|
||||
'source.revision': model.sourceRevision.orNull,
|
||||
'source.revisionStatus': model.sourceRevisionStatus.orNull,
|
||||
'source.license': model.sourceLicense.orNull,
|
||||
'source.licenseUri': model.sourceLicenseUri.orNull,
|
||||
'source.attribution': model.sourceAttribution.orNull,
|
||||
'source.verificationDate': model.sourceVerificationDate.orNull,
|
||||
'transformations.summary': model.transformationsSummary.orNull]
|
||||
required.each { String key, String value ->
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new GradleException("Required model metadata is missing: ${key}")
|
||||
}
|
||||
}
|
||||
validateRevisionMetadata(model.sourceRevision.get(), model.sourceRevisionStatus.get())
|
||||
}
|
||||
|
||||
/** Accepts an exact recorded revision or the explicit legacy-import sentinel, but never an absent status. */
|
||||
static void validateRevisionMetadata(final String revision, final String status) {
|
||||
if (revision == null || revision.isBlank()) {
|
||||
throw new GradleException('Required model metadata is missing: source.revision')
|
||||
}
|
||||
if (status == null || status.isBlank()) {
|
||||
throw new GradleException('Required model metadata is missing: source.revisionStatus')
|
||||
}
|
||||
final String sentinel = 'not-recorded-in-legacy-import'
|
||||
if (revision == sentinel && status != sentinel) {
|
||||
throw new GradleException('The legacy revision sentinel requires source.revisionStatus=not-recorded-in-legacy-import.')
|
||||
}
|
||||
if (revision != sentinel && status != 'recorded') {
|
||||
throw new GradleException('An exact source revision requires source.revisionStatus=recorded.')
|
||||
}
|
||||
}
|
||||
|
||||
/** Validates the model-specific attribution and ShareAlike notice. */
|
||||
static void validateShareAlikeNotice(final File notice, final RadixorModelExtension model) {
|
||||
validateShareAlikeNoticeText(notice.getText('UTF-8'), notice.toString(), model.modelId.get(),
|
||||
model.sourceRepository.get(), model.sourceLicenseUri.get(), model.sourceRevision.get(),
|
||||
model.sourceRevisionStatus.get())
|
||||
}
|
||||
|
||||
/** Validates required content in one UniMorph model-data notice. */
|
||||
static void validateShareAlikeNoticeText(final String text, final String noticeName,
|
||||
final String modelId, final String repository, final String licenseUri,
|
||||
final String revision, final String revisionStatus) {
|
||||
final List<String> required = [
|
||||
"Model ID: ${modelId}",
|
||||
"Official repository: ${repository}",
|
||||
'Attribution:',
|
||||
'License:\nCreative Commons Attribution-ShareAlike 3.0 Unported',
|
||||
"Canonical license URI: ${licenseUri}",
|
||||
'Radixor modifications:',
|
||||
"Revision status: ${revisionStatus}",
|
||||
'Copyright (C) 2026, Leo Galambos.',
|
||||
'Radixor-specific selection, verification, cleaning, normalization,',
|
||||
'to the extent protected by applicable law.',
|
||||
'The underlying morphological data remains attributed to UniMorph and',
|
||||
"This derived model data, including Radixor's protectable contributions,",
|
||||
'is distributed under Creative Commons Attribution-ShareAlike 3.0',
|
||||
'Neither UniMorph nor any upstream contributor endorses Radixor.']
|
||||
if (revision == 'not-recorded-in-legacy-import') {
|
||||
required.add('The exact UniMorph commit used for the original Radixor import was not recorded.')
|
||||
}
|
||||
final List<String> missing = required.findAll { String value -> !text.contains(value) }
|
||||
if (!missing.isEmpty()) {
|
||||
throw new GradleException("Model notice ${noticeName} is missing required content: ${missing.join(', ')}")
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects generic license files and foreign notices in a UniMorph model artifact. */
|
||||
static void validateUniMorphJarContents(final List<String> names) {
|
||||
if (names.any { String name -> name.startsWith('META-INF/LICENSES/') }) {
|
||||
throw new GradleException('A UniMorph model artifact must use only its model-specific notice for data licensing.')
|
||||
}
|
||||
if (names.count { String name -> name.startsWith('META-INF/NOTICE/') && !name.endsWith('/') } != 1) {
|
||||
throw new GradleException('A UniMorph model artifact must contain exactly one model-specific notice.')
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects UniMorph licensing material in the separately licensed PoliMorf artifact. */
|
||||
static void validatePoliMorfJarContents(final List<String> names) {
|
||||
if (names.any { String name -> name.startsWith('META-INF/NOTICE/')
|
||||
|| name.contains('CC-BY-SA') }) {
|
||||
throw new GradleException('The PoliMorf artifact must not contain UniMorph CC BY-SA material.')
|
||||
}
|
||||
}
|
||||
|
||||
/** Memory-bounded validation statistics for one dictionary input. */
|
||||
static final class DictionaryValidationResult {
|
||||
final long acceptedGroupCount
|
||||
final long acceptedFormCount
|
||||
final long ignoredEmptyVariantCount
|
||||
|
||||
DictionaryValidationResult(final long acceptedGroupCount, final long acceptedFormCount,
|
||||
final long ignoredEmptyVariantCount) {
|
||||
this.acceptedGroupCount = acceptedGroupCount
|
||||
this.acceptedFormCount = acceptedFormCount
|
||||
this.ignoredEmptyVariantCount = ignoredEmptyVariantCount
|
||||
}
|
||||
}
|
||||
|
||||
/** Validates GZip, strict UTF-8, and dictionary rows without retaining decompressed input. */
|
||||
static DictionaryValidationResult validateDictionary(final File file) {
|
||||
long acceptedGroups = 0L
|
||||
long acceptedForms = 0L
|
||||
long ignoredEmptyVariants = 0L
|
||||
try {
|
||||
final def decoder = StandardCharsets.UTF_8.newDecoder()
|
||||
.onMalformedInput(CodingErrorAction.REPORT)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPORT)
|
||||
Files.newInputStream(file.toPath()).withCloseable { InputStream source ->
|
||||
new BufferedInputStream(source).withCloseable { BufferedInputStream bufferedInput ->
|
||||
new GZIPInputStream(bufferedInput).withCloseable { GZIPInputStream gzipInput ->
|
||||
new BufferedReader(new InputStreamReader(gzipInput, decoder)).withCloseable { BufferedReader reader ->
|
||||
String line
|
||||
long lineNumber = 0L
|
||||
while ((line = reader.readLine()) != null) {
|
||||
lineNumber++
|
||||
final String trimmed = line.trim()
|
||||
if (trimmed && !trimmed.startsWith('#') && !trimmed.startsWith('//')) {
|
||||
final String[] columns = line.split('\\t', -1)
|
||||
if (columns[0].isEmpty()) {
|
||||
throw new GradleException("Invalid Radixor dictionary row ${lineNumber} in ${file}.")
|
||||
}
|
||||
if (containsUnicodeWhitespace(columns[0])) continue
|
||||
long acceptedRowForms = 1L
|
||||
for (int index = 1; index < columns.length; index++) {
|
||||
final String variant = columns[index]
|
||||
if (variant.isEmpty()) {
|
||||
ignoredEmptyVariants++
|
||||
} else if (!containsUnicodeWhitespace(variant)) {
|
||||
acceptedRowForms++
|
||||
}
|
||||
}
|
||||
acceptedGroups++
|
||||
acceptedForms += acceptedRowForms
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (GradleException exception) {
|
||||
throw exception
|
||||
} catch (Exception exception) {
|
||||
throw new GradleException("Invalid GZip or UTF-8 model input: ${file}", exception)
|
||||
}
|
||||
if (acceptedGroups == 0L) throw new GradleException("Model dictionary contains no valid rows: ${file}")
|
||||
if (ignoredEmptyVariants > 0L) {
|
||||
println("Model validation warning: " + file + " contains " + ignoredEmptyVariants
|
||||
+ " empty variant columns; the production parser intentionally ignores empty variants.")
|
||||
}
|
||||
return new DictionaryValidationResult(acceptedGroups, acceptedForms, ignoredEmptyVariants)
|
||||
}
|
||||
|
||||
/** Detects Unicode whitespace in one bounded dictionary field. */
|
||||
private static boolean containsUnicodeWhitespace(final String value) {
|
||||
for (int index = 0; index < value.length(); index++) {
|
||||
if (Character.isWhitespace(value.charAt(index))) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** Builds deterministic descriptor text. */
|
||||
private static String descriptorText(final RadixorModelExtension model, final String version,
|
||||
final String resource, final String checksum) {
|
||||
return """model.id=${model.modelId.get()}
|
||||
model.version=${version}
|
||||
model.language=${model.language.get()}
|
||||
model.displayName=${model.displayName.get()}
|
||||
model.resource=${resource}
|
||||
model.default=${model.defaultModel.get()}
|
||||
model.format=radixor-dictionary-tsv-gzip
|
||||
model.formatVersion=1
|
||||
model.sha256=${checksum}
|
||||
model.rightToLeft=${['FA_IR', 'HE_IL', 'YI'].contains(model.language.get())}
|
||||
model.caseProcessing=LOWERCASE_WITH_LOCALE_ROOT
|
||||
model.diacriticProcessing=AS_IS
|
||||
model.storeOriginal=true
|
||||
source.name=${model.sourceName.get()}
|
||||
source.version=${model.sourceVersion.get()}
|
||||
source.project=${model.sourceProject.get()}
|
||||
source.repository=${model.sourceRepository.get()}
|
||||
source.dataset=${model.sourceDataset.get()}
|
||||
source.revision=${model.sourceRevision.get()}
|
||||
source.revisionStatus=${model.sourceRevisionStatus.get()}
|
||||
source.license=${model.sourceLicense.get()}
|
||||
source.licenseUri=${model.sourceLicenseUri.get()}
|
||||
source.attribution=${model.sourceAttribution.get()}
|
||||
source.verificationDate=${model.sourceVerificationDate.get()}
|
||||
transformations.summary=${model.transformationsSummary.get()}
|
||||
compiler.radixorVersion=3.x
|
||||
compiler.radixorCommit=unavailable
|
||||
statistics.groups=unavailable
|
||||
statistics.forms=unavailable
|
||||
"""
|
||||
}
|
||||
|
||||
/** Calculates the lowercase hexadecimal SHA-256 digest. */
|
||||
private static String sha256(final File file) {
|
||||
return sha256(file.bytes)
|
||||
}
|
||||
|
||||
/** Calculates the lowercase hexadecimal SHA-256 digest of bytes. */
|
||||
private static String sha256(final byte[] bytes) {
|
||||
return MessageDigest.getInstance('SHA-256').digest(bytes).collect { byte value -> String.format('%02x', value & 0xff) }.join()
|
||||
}
|
||||
|
||||
/** Calculates a lowercase hexadecimal digest using the requested algorithm. */
|
||||
private static String sha256WithAlgorithm(final File file, final String algorithm) {
|
||||
return MessageDigest.getInstance(algorithm).digest(file.bytes)
|
||||
.collect { byte value -> String.format('%02x', value & 0xff) }.join()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.GradleException
|
||||
import org.gradle.api.file.RegularFileProperty
|
||||
import org.gradle.api.provider.MapProperty
|
||||
import org.gradle.api.provider.Property
|
||||
import org.gradle.api.tasks.Input
|
||||
import org.gradle.api.tasks.InputFile
|
||||
import org.gradle.api.tasks.Optional
|
||||
import org.gradle.api.tasks.PathSensitive
|
||||
import org.gradle.api.tasks.PathSensitivity
|
||||
import org.gradle.api.tasks.TaskAction
|
||||
|
||||
/** Validates one immutable model input without retaining Project state. */
|
||||
abstract class ValidateModelInputTask extends DefaultTask {
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getDictionaryFile()
|
||||
@InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getVersionFile()
|
||||
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getLicenseFile()
|
||||
@Optional @InputFile @PathSensitive(PathSensitivity.RELATIVE) abstract RegularFileProperty getNoticeFile()
|
||||
@Input abstract Property<String> getModelId()
|
||||
@Input abstract Property<String> getModuleName()
|
||||
@Input abstract Property<Boolean> getShareAlike()
|
||||
@Input abstract MapProperty<String, String> getMetadata()
|
||||
|
||||
/** Performs deterministic metadata, licensing, and streaming dictionary validation. */
|
||||
@TaskAction
|
||||
void validateInput() {
|
||||
final File dictionary = dictionaryFile.get().asFile
|
||||
final String id = modelId.get()
|
||||
final String version = versionFile.get().asFile.getText('UTF-8').trim()
|
||||
if (id != moduleName.get() || !(id ==~ /[a-z]{2}(?:-[a-z]{2})?-[a-z0-9]+(?:-[a-z0-9]+)*/)) {
|
||||
throw new GradleException("Model ID '${id}' must equal module '${moduleName.get()}' and use the safe model-ID syntax.")
|
||||
}
|
||||
if (!(version ==~ /[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?/)) {
|
||||
throw new GradleException("Invalid semantic model version '${version}'.")
|
||||
}
|
||||
final Map<String, String> values = metadata.get()
|
||||
values.each { String key, String value ->
|
||||
if (value == null || value.isBlank()) throw new GradleException("Required model metadata is missing: ${key}")
|
||||
}
|
||||
RadixorModelPlugin.validateRevisionMetadata(values['source.revision'], values['source.revisionStatus'])
|
||||
if (shareAlike.get()) {
|
||||
final File notice = noticeFile.get().asFile
|
||||
RadixorModelPlugin.validateShareAlikeNoticeText(notice.getText('UTF-8'), notice.toString(), id,
|
||||
values['source.repository'], values['source.licenseUri'], values['source.revision'],
|
||||
values['source.revisionStatus'])
|
||||
} else {
|
||||
final String text = licenseFile.get().asFile.getText('UTF-8')
|
||||
if (!text.contains('SPDX-License-Identifier: BSD-2-Clause')
|
||||
|| !text.contains('Copyright (c) 2016, Marcin Miłkowski')) {
|
||||
throw new GradleException('The PoliMorf license must contain the complete BSD-2-Clause text and upstream attribution.')
|
||||
}
|
||||
}
|
||||
RadixorModelPlugin.validateDictionary(dictionary)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package org.egothor.radixor
|
||||
|
||||
import org.gradle.api.GradleException
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.io.TempDir
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.zip.GZIPOutputStream
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue
|
||||
|
||||
/** Tests model licensing metadata and packaged-resource validation boundaries. */
|
||||
final class RadixorModelPluginTest {
|
||||
@TempDir
|
||||
Path temporaryDirectory
|
||||
|
||||
/** Accepts a known exact source revision. */
|
||||
@Test
|
||||
void acceptsKnownExactRevision() {
|
||||
RadixorModelPlugin.validateRevisionMetadata('6e63b53', 'recorded')
|
||||
}
|
||||
|
||||
/** Accepts the explicit legacy-import sentinel without fabricating a revision. */
|
||||
@Test
|
||||
void acceptsUnknownLegacyRevision() {
|
||||
RadixorModelPlugin.validateRevisionMetadata(
|
||||
'not-recorded-in-legacy-import', 'not-recorded-in-legacy-import')
|
||||
}
|
||||
|
||||
/** Rejects a missing revision-status declaration. */
|
||||
@Test
|
||||
void rejectsMissingRevisionStatus() {
|
||||
assertThrows(GradleException) {
|
||||
RadixorModelPlugin.validateRevisionMetadata('6e63b53', '')
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects a missing model-specific notice input. */
|
||||
@Test
|
||||
void rejectsMissingLicensingInputs() {
|
||||
File missing = new File('build/nonexistent-model-licensing-input')
|
||||
assertThrows(GradleException) {
|
||||
RadixorModelPlugin.requireFile(missing, 'Required model notice is missing')
|
||||
}
|
||||
}
|
||||
|
||||
/** Accepts a complete model-specific UniMorph notice. */
|
||||
@Test
|
||||
void acceptsCompleteUniMorphNotice() {
|
||||
validateNotice(validNotice())
|
||||
}
|
||||
|
||||
/** Rejects each independently required notice statement. */
|
||||
@Test
|
||||
void rejectsIncompleteUniMorphNotices() {
|
||||
[
|
||||
'Copyright (C) 2026, Leo Galambos.',
|
||||
'Attribution:',
|
||||
'Creative Commons Attribution-ShareAlike 3.0 Unported',
|
||||
'Canonical license URI:',
|
||||
"This derived model data, including Radixor's protectable contributions,",
|
||||
'Radixor modifications:',
|
||||
'Revision status:',
|
||||
'Neither UniMorph nor any upstream contributor endorses Radixor.'
|
||||
].each { String required ->
|
||||
assertThrows(GradleException) {
|
||||
validateNotice(validNotice().replace(required, 'omitted'))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects packaged notice bytes that differ from their model-module source. */
|
||||
@Test
|
||||
void rejectsIncorrectPackagedNotice() {
|
||||
assertThrows(GradleException) {
|
||||
RadixorModelPlugin.requireMatchingChecksum(
|
||||
'notice', 'META-INF/NOTICE/test-model-data.txt', 'source', 'different')
|
||||
}
|
||||
}
|
||||
|
||||
/** Rejects UniMorph CC material in the separately licensed PoliMorf artifact. */
|
||||
@Test
|
||||
void rejectsUniMorphMaterialInPoliMorf() {
|
||||
assertThrows(GradleException) {
|
||||
RadixorModelPlugin.validatePoliMorfJarContents(
|
||||
['META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt', 'META-INF/NOTICE/test-data.txt'])
|
||||
}
|
||||
assertThrows(GradleException) {
|
||||
RadixorModelPlugin.validatePoliMorfJarContents(
|
||||
['META-INF/LICENSES/PoliMorf-BSD-2-Clause.txt', 'META-INF/LICENSES/CC-BY-SA-3.0.txt'])
|
||||
}
|
||||
}
|
||||
|
||||
/** Streams a large dictionary while retaining only aggregate counters and the current row. */
|
||||
@Test
|
||||
void validatesLargeDictionaryWithBoundedState() {
|
||||
final int groups = 250_000
|
||||
final File dictionary = temporaryDirectory.resolve('large.gz').toFile()
|
||||
writeGzip(dictionary) { BufferedWriter writer ->
|
||||
for (int index = 0; index < groups; index++) {
|
||||
writer.write("stem${index}\tvariant${index}\t\n")
|
||||
}
|
||||
}
|
||||
|
||||
final RadixorModelPlugin.DictionaryValidationResult result =
|
||||
RadixorModelPlugin.validateDictionary(dictionary)
|
||||
|
||||
assertEquals(groups, result.acceptedGroupCount)
|
||||
assertEquals(groups * 2L, result.acceptedFormCount)
|
||||
assertEquals(groups, result.ignoredEmptyVariantCount)
|
||||
}
|
||||
|
||||
/** Rejects a source that is not a GZip stream. */
|
||||
@Test
|
||||
void rejectsInvalidGzip() {
|
||||
final File dictionary = temporaryDirectory.resolve('invalid.gz').toFile()
|
||||
Files.writeString(dictionary.toPath(), 'not gzip', StandardCharsets.UTF_8)
|
||||
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
|
||||
}
|
||||
|
||||
/** Rejects malformed UTF-8 through the strict incremental decoder. */
|
||||
@Test
|
||||
void rejectsMalformedUtf8() {
|
||||
final File dictionary = temporaryDirectory.resolve('malformed-utf8.gz').toFile()
|
||||
new GZIPOutputStream(Files.newOutputStream(dictionary.toPath())).withCloseable { OutputStream output ->
|
||||
output.write([0x73, 0x74, 0x65, 0x6d, 0x09, 0xc3, 0x28, 0x0a] as byte[])
|
||||
}
|
||||
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
|
||||
}
|
||||
|
||||
/** Rejects structurally invalid rows with an empty stem. */
|
||||
@Test
|
||||
void rejectsInvalidRows() {
|
||||
final File dictionary = temporaryDirectory.resolve('invalid-row.gz').toFile()
|
||||
writeGzip(dictionary) { BufferedWriter writer -> writer.write("\tvariant\n") }
|
||||
assertThrows(GradleException) { RadixorModelPlugin.validateDictionary(dictionary) }
|
||||
}
|
||||
|
||||
/** Preserves the production parser policy for Unicode-whitespace items. */
|
||||
@Test
|
||||
void rejectsUnicodeWhitespaceItemsWithoutRejectingTheSource() {
|
||||
final File dictionary = temporaryDirectory.resolve('whitespace-items.gz').toFile()
|
||||
writeGzip(dictionary) { BufferedWriter writer ->
|
||||
writer.write("invalid stem\tvariant\n")
|
||||
writer.write("valid\taccepted\tinvalid variant\n")
|
||||
}
|
||||
final RadixorModelPlugin.DictionaryValidationResult result =
|
||||
RadixorModelPlugin.validateDictionary(dictionary)
|
||||
assertEquals(1L, result.acceptedGroupCount)
|
||||
assertEquals(2L, result.acceptedFormCount)
|
||||
}
|
||||
|
||||
/** Streams the complete maintained PoliMorf model input successfully. */
|
||||
@Test
|
||||
void validatesFullPoliMorfInput() {
|
||||
final List<File> candidates = [
|
||||
new File('models/pl-pl-polimorf/src/modelInput/stemmer.gz'),
|
||||
new File('../models/pl-pl-polimorf/src/modelInput/stemmer.gz')]
|
||||
final File dictionary = candidates.find { File candidate -> candidate.isFile() }
|
||||
assertTrue(dictionary != null, 'The complete PoliMorf model input must be available to build-logic tests.')
|
||||
|
||||
final RadixorModelPlugin.DictionaryValidationResult result =
|
||||
RadixorModelPlugin.validateDictionary(dictionary)
|
||||
assertTrue(result.acceptedGroupCount > 0L)
|
||||
assertTrue(result.acceptedFormCount > result.acceptedGroupCount)
|
||||
}
|
||||
|
||||
private static void writeGzip(final File target, final Closure<Void> content) {
|
||||
new GZIPOutputStream(Files.newOutputStream(target.toPath())).withCloseable { OutputStream gzip ->
|
||||
new BufferedWriter(new OutputStreamWriter(gzip, StandardCharsets.UTF_8)).withCloseable {
|
||||
BufferedWriter writer -> content.call(writer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void validateNotice(final String text) {
|
||||
RadixorModelPlugin.validateShareAlikeNoticeText(text, 'test notice', 'test-model',
|
||||
'https://github.com/unimorph/test', 'https://creativecommons.org/licenses/by-sa/3.0/',
|
||||
'not-recorded-in-legacy-import', 'not-recorded-in-legacy-import')
|
||||
}
|
||||
|
||||
private static String validNotice() {
|
||||
return '''Model ID: test-model
|
||||
Official repository: https://github.com/unimorph/test
|
||||
Attribution: UniMorph and upstream contributors
|
||||
License:
|
||||
Creative Commons Attribution-ShareAlike 3.0 Unported
|
||||
Canonical license URI: https://creativecommons.org/licenses/by-sa/3.0/
|
||||
Radixor modifications: Cleaning and packaging.
|
||||
Revision status: not-recorded-in-legacy-import
|
||||
The exact UniMorph commit used for the original Radixor import was not recorded.
|
||||
Copyright (C) 2026, Leo Galambos.
|
||||
Radixor-specific selection, verification, cleaning, normalization,
|
||||
to the extent protected by applicable law.
|
||||
The underlying morphological data remains attributed to UniMorph and
|
||||
This derived model data, including Radixor's protectable contributions,
|
||||
is distributed under Creative Commons Attribution-ShareAlike 3.0
|
||||
Neither UniMorph nor any upstream contributor endorses Radixor.
|
||||
'''
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user