Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat!: modularize stemmer models and release infrastructure

Browse Source 
Move bundled stemmer dictionaries from the core artifact into independently
versioned model modules. Add model discovery and explicit model-loading APIs,
a standard model aggregate, a model BOM, and dedicated model and catalog
release workflows.

Add full PoliMorf integration, model provenance and licensing validation,
streaming model-input verification, strict dependency verification, consumer
resolution tests, Configuration Cache compatibility, and expanded JMH,
quality, documentation, and release checks.

Upgrade the CycloneDX and JMH Gradle plugins and remove Gradle 10 and Java
compiler deprecations.

BREAKING CHANGE: The core Radixor artifact no longer contains bundled stemmer
dictionaries. Applications must add the required model artifacts, the standard
model aggregate, or model dependencies managed through the Radixor model BOM.
This commit is contained in:
Leo Galambos
2026-07-22 23:33:28 +02:00
parent 9c5b9e331b
commit e7800b29c9
250 changed files with 7297 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkCorpusSupport.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/jmh/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequence.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/jmh/java/org/egothor/stemmer/benchmark/EnglishComparisonCorpus.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

3
src/jmh/java/org/egothor/stemmer/benchmark/EnglishRadixorDictionaryCoverageBenchmark.java
View File

@@ -331,7 +331,8 @@ public class EnglishRadixorDictionaryCoverageBenchmark {
}
private static List<DictionaryRow> readEnglishRows() throws IOException {
final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath();
final String resourcePath = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
.requireDefault(StemmerPatchTrieLoader.Language.US_UK).resource();
final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath);
if (resource == null) {
throw new IllegalStateException("Missing bundled English dictionary resource " + resourcePath + ".");

10
src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonBenchmark.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/jmh/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStream.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieCompilationBenchmark.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

3
src/jmh/java/org/egothor/stemmer/benchmark/LanguageBenchmarkCorpus.java
View File

@@ -332,7 +332,8 @@ final class LanguageBenchmarkCorpus {
*/
private static List<Entry> readCandidates(final StemmerPatchTrieLoader.Language language, final int maximumTokenCount)
throws IOException {
final String resourcePath = language.resourcePath();
final String resourcePath = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
.requireDefault(language).resource();
final InputStream resource = StemmerPatchTrieLoader.class.getClassLoader().getResourceAsStream(resourcePath);
if (resource == null) {
throw new IllegalStateException("Missing bundled benchmark resource " + resourcePath + ".");

2
src/jmh/java/org/egothor/stemmer/benchmark/MultiLanguageStemmerComparisonBenchmark.java
View File

@@ -95,7 +95,7 @@ import morfologik.stemming.WordData;
* Each benchmark operation processes the same changed-token dictionary corpus
* for one language, repeated only when the changed-token resource contains
* fewer than 5,000 token fields. The token corpus is built during trial setup
* from Radixor's bundled dictionary for that same language. Lucene TokenFilter
* from Radixor's registered default-model dictionary for that same language. Lucene TokenFilter
* methods include TokenStream and attribute overhead; direct Stempel measures
* the public table-driven stemmer API without TokenFilter overhead.
* </p>

30
src/jmh/java/org/egothor/stemmer/benchmark/QualityStemmerMatrix.java
View File

@@ -1,3 +1,33 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import java.io.IOException;

10
src/jmh/java/org/egothor/stemmer/benchmark/SnowballStemmerAdapter.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

1
src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java
View File

@@ -444,6 +444,7 @@ public class StemmerComparisonBenchmarkQuality {
* @return quality evaluator
* @throws IOException if stemmer resources cannot be loaded
*/
@SuppressWarnings("deprecation") // Lucene retains SpanishMinimalStemFilter only for compatibility benchmarking.
CandidateStemmer createStemmer() throws IOException {
if (name().endsWith("_RADIXOR")) {
return radixor(createRadixorStemmer(this.radixorLanguage));

10
src/jmh/java/org/egothor/stemmer/benchmark/package-info.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

30
src/main/java/module-info.java
View File

@@ -1,3 +1,33 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
module org.egothor.radixor {
requires java.logging;

10
src/main/java/org/egothor/stemmer/CaseProcessingMode.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/Compile.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/DiacriticProcessingMode.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/DiacriticStripper.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

46
src/main/java/org/egothor/stemmer/DuplicateStemmerModelException.java Normal file
View File

@@ -0,0 +1,46 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
/**
* Indicates that two classpath descriptors declare the same stable model identifier.
* Classpath order cannot resolve this conflict because model IDs must be unique.
*/
public final class DuplicateStemmerModelException extends IllegalStateException {
private static final long serialVersionUID = 1L;
/**
* Creates an exception describing the conflicting descriptor locations.
* @param message diagnostic containing both descriptor locations
*/
public DuplicateStemmerModelException(final String message) {
super(message);
}
}

10
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/PatchCommandEncoder.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/ReductionMode.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

12
src/main/java/org/egothor/stemmer/ReductionSettings.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -38,7 +38,7 @@ import java.util.Objects;
* <p>
* The settings influence how mutable trie nodes are merged into canonical
* read-only nodes during compilation.
*
*
* @param reductionMode reduction mode
* @param dominantWinnerMinPercent minimum dominant winner percentage
* @param dominantWinnerOverSecondRatio minimum winner-over-second ratio

10
src/main/java/org/egothor/stemmer/StemmerDictionaryParser.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

28
src/main/java/org/egothor/stemmer/StemmerKnowledgeExperiment.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -107,11 +107,11 @@ public final class StemmerKnowledgeExperiment {
}
/**
* Evaluates all supported bundled dictionaries using the supplied seed.
* Evaluates all supported registered default-model dictionaries using the supplied seed.
*
* @param seed deterministic sampling seed
* @return immutable ordered list of experiment rows
* @throws IOException if reading a bundled dictionary fails
* @throws IOException if reading a registered default dictionary fails
*/
public List<ResultRow> evaluateAllBundledLanguages(final long seed) throws IOException {
final List<ResultRow> rows = new ArrayList<>();
@@ -122,19 +122,19 @@ public final class StemmerKnowledgeExperiment {
}
/**
* Evaluates one bundled dictionary across all supported experiment
* Evaluates one registered default-model dictionary across all supported experiment
* configurations.
*
* @param language bundled language dictionary
* @param language language whose default model dictionary is evaluated
* @param seed deterministic sampling seed
* @return immutable ordered list of experiment rows
* @throws NullPointerException if {@code language} is {@code null}
* @throws IOException if reading the bundled dictionary fails
* @throws IOException if reading the registered dictionary fails
*/
public List<ResultRow> evaluateBundledLanguage(final StemmerPatchTrieLoader.Language language, final long seed)
throws IOException {
Objects.requireNonNull(language, "language");
final String resourcePath = language.resourcePath();
final String resourcePath = StemmerModelRegistry.fromContextClassLoader().requireDefault(language).resource();
try (InputStream inputStream = StemmerPatchTrieLoader.openBundledResource(resourcePath)) {
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
@@ -349,13 +349,13 @@ public final class StemmerKnowledgeExperiment {
* @param trie compiled trie under test
* @return immutable counts for this single input
*/
@SuppressWarnings("deprecation")
private static EvaluationCounts evaluateInput(final String input, final String expectedStem,
final FrequencyTrie<String> trie) {
long getCorrect = 0L;
final String preferredPatch = trie.get(input);
if (preferredPatch != null) {
final String preferredStem = PatchCommandEncoder.apply(input, preferredPatch);
final String preferredStem = PatchCommandEncoder.compile(preferredPatch, trie.traversalDirection())
.apply(input);
if (expectedStem.equals(preferredStem)) {
getCorrect = 1L;
}
@@ -371,7 +371,7 @@ public final class StemmerKnowledgeExperiment {
long falsePositives = 0L;
long coveredInputs = 0L;
for (String patch : patches) {
final String candidateStem = PatchCommandEncoder.apply(input, patch);
final String candidateStem = PatchCommandEncoder.compile(patch, trie.traversalDirection()).apply(input);
if (expectedStem.equals(candidateStem)) {
truePositives++;
coveredInputs = 1L;

10
src/main/java/org/egothor/stemmer/StemmerKnowledgeExperimentCli.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

109
src/main/java/org/egothor/stemmer/StemmerModelDescriptor.java Normal file
View File

@@ -0,0 +1,109 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import java.net.URL;
import java.util.Objects;
/**
* Immutable validated metadata for one independently versioned Radixor model.
*
* <p>The descriptor identifies a GZip-compressed Radixor textual dictionary. It
* does not contain a precompiled trie. Instances are created during deterministic
* registry discovery and retain the descriptor URL used in diagnostics.</p>
*/
@SuppressWarnings({ "PMD.DataClass", "PMD.ExcessiveParameterList", "PMD.CommentDefaultAccessModifier" })
public final class StemmerModelDescriptor implements Comparable<StemmerModelDescriptor> {
private final String id;
private final String version;
private final StemmerPatchTrieLoader.Language language;
private final String displayName;
private final String resource;
private final boolean defaultModel;
private final String format;
private final int formatVersion;
private final String sha256;
private final URL source;
private final ClassLoader classLoader;
/** Creates a validated immutable descriptor. */
StemmerModelDescriptor(final String id, final String version, final StemmerPatchTrieLoader.Language language,
final String displayName, final String resource, final boolean defaultModel, final String format,
final int formatVersion, final String sha256, final URL source, final ClassLoader classLoader) {
this.id = Objects.requireNonNull(id, "id");
this.version = Objects.requireNonNull(version, "version");
this.language = Objects.requireNonNull(language, "language");
this.displayName = Objects.requireNonNull(displayName, "displayName");
this.resource = Objects.requireNonNull(resource, "resource");
this.defaultModel = defaultModel;
this.format = Objects.requireNonNull(format, "format");
this.formatVersion = formatVersion;
this.sha256 = Objects.requireNonNull(sha256, "sha256");
this.source = Objects.requireNonNull(source, "source");
this.classLoader = Objects.requireNonNull(classLoader, "classLoader");
}
/** Returns the stable model identifier used for explicit deterministic selection. */
public String id() { return this.id; }
/** Returns the independently managed model artifact version. */
public String version() { return this.version; }
/** Returns the represented language. */
public StemmerPatchTrieLoader.Language language() { return this.language; }
/** Returns the human-readable model name. */
public String displayName() { return this.displayName; }
/** Returns the namespaced classpath dictionary resource. */
public String resource() { return this.resource; }
/**
* Returns whether model metadata declares this model as a default candidate.
* Language-oriented runtime resolution uses
* {@link StemmerPatchTrieLoader.Language#defaultModelId()} as its authoritative
* mapping.
*/
public boolean isDefaultModel() { return this.defaultModel; }
/** Returns the dictionary format identifier. */
public String format() { return this.format; }
/** Returns the dictionary format version. */
public int formatVersion() { return this.formatVersion; }
/** Returns the lowercase SHA-256 digest of the compressed runtime resource bytes. */
public String sha256() { return this.sha256; }
/** Returns the descriptor source URL used for diagnostics. */
public URL source() { return this.source; }
/** Returns the discovering class loader. */
ClassLoader classLoader() { return this.classLoader; }
/** Compares descriptors by stable model identifier. */
@Override
public int compareTo(final StemmerModelDescriptor other) { return this.id.compareTo(other.id); }
/** Returns a concise descriptor representation. */
@Override
public String toString() { return this.id + "@" + this.version + " (" + this.source + ")"; }
}

56
src/main/java/org/egothor/stemmer/StemmerModelIntegrityException.java Normal file
View File

@@ -0,0 +1,56 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
/**
* Indicates invalid model packaging or integrity, including malformed metadata,
* invalid index entries, missing resources, inconsistent language declarations,
* or a checksum mismatch.
*/
public final class StemmerModelIntegrityException extends IllegalStateException {
private static final long serialVersionUID = 1L;
/**
* Creates an exception describing an integrity failure.
* @param message precise integrity diagnostic
*/
public StemmerModelIntegrityException(final String message) {
super(message);
}
/**
* Creates an exception describing an integrity failure and its cause.
* @param message precise integrity diagnostic
* @param cause underlying parsing or platform failure
*/
public StemmerModelIntegrityException(final String message, final Throwable cause) {
super(message, cause);
}
}

46
src/main/java/org/egothor/stemmer/StemmerModelNotFoundException.java Normal file
View File

@@ -0,0 +1,46 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
/**
* Indicates that an explicitly requested model or a language's documented default
* model is absent from the selected class loader's runtime classpath.
*/
public final class StemmerModelNotFoundException extends IllegalStateException {
private static final long serialVersionUID = 1L;
/**
* Creates an exception with a precise remediation message.
* @param message diagnostic naming the missing ID and suggested dependency
*/
public StemmerModelNotFoundException(final String message) {
super(message);
}
}

228
src/main/java/org/egothor/stemmer/StemmerModelRegistry.java Normal file
View File

@@ -0,0 +1,228 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
/**
* Immutable deterministic registry of models discovered from classpath indexes.
*
* <p>Discovery enumerates every {@value #INDEX_RESOURCE} visible to the selected
* class loader, validates the referenced descriptors, sorts them by stable model
* identifier, and rejects duplicate identifiers. Selection never depends on
* classpath order. Registry creation validates metadata and resource presence;
* {@link StemmerPatchTrieLoader} verifies resource bytes when loading a model.</p>
*
* <p>Registry creation is not globally cached. Applications should normally
* discover once for a class-loader scope and retain the immutable result.</p>
*/
@SuppressWarnings({ "PMD.UseProperClassLoader", "PMD.ControlStatementBraces" })
public final class StemmerModelRegistry {
/** Fixed classpath index name used by every model artifact. */
public static final String INDEX_RESOURCE = "META-INF/radixor/models.index";
private static final String FORMAT = "radixor-dictionary-tsv-gzip";
private static final int FORMAT_VERSION = 1;
private final Map<String, StemmerModelDescriptor> descriptors;
/** Creates an immutable registry from already validated descriptors. */
private StemmerModelRegistry(final Map<String, StemmerModelDescriptor> descriptors) {
this.descriptors = Collections.unmodifiableMap(new LinkedHashMap<>(descriptors));
}
/**
* Discovers models through the current thread context class loader.
*
* <p>If the context loader is {@code null}, the defining class loader of this
* registry is used.</p>
*
* @return immutable registry in stable model-ID order
* @throws IOException if index or descriptor resources cannot be enumerated or read
* @throws DuplicateStemmerModelException if two descriptors declare one model ID
* @throws StemmerModelIntegrityException if an index, descriptor, or declared resource is invalid
* @throws UnsupportedStemmerModelFormatException if a descriptor uses an unsupported format
*/
public static StemmerModelRegistry fromContextClassLoader() throws IOException {
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
return fromClassLoader(classLoader == null ? StemmerModelRegistry.class.getClassLoader() : classLoader);
}
/**
* Discovers and validates every indexed descriptor visible to an explicit class loader.
*
* @param classLoader class loader whose indexed model resources are visible
* @return immutable registry in stable model-ID order
* @throws NullPointerException if {@code classLoader} is {@code null}
* @throws IOException if index or descriptor resources cannot be enumerated or read
* @throws DuplicateStemmerModelException if two descriptors declare one model ID
* @throws StemmerModelIntegrityException if an index, descriptor, or declared resource is invalid
* @throws UnsupportedStemmerModelFormatException if a descriptor uses an unsupported format
*/
public static StemmerModelRegistry fromClassLoader(final ClassLoader classLoader) throws IOException {
Objects.requireNonNull(classLoader, "classLoader");
final List<URL> indexes = Collections.list(classLoader.getResources(INDEX_RESOURCE));
indexes.sort((left, right) -> left.toExternalForm().compareTo(right.toExternalForm()));
final List<StemmerModelDescriptor> discovered = new ArrayList<>();
for (URL index : indexes) {
readIndex(index, classLoader, discovered);
}
Collections.sort(discovered);
final Map<String, StemmerModelDescriptor> byId = new LinkedHashMap<>();
for (StemmerModelDescriptor descriptor : discovered) {
final StemmerModelDescriptor previous = byId.putIfAbsent(descriptor.id(), descriptor);
if (previous != null) {
throw new DuplicateStemmerModelException("Duplicate model ID '" + descriptor.id() + "' at "
+ previous.source() + " and " + descriptor.source() + ".");
}
}
return new StemmerModelRegistry(byId);
}
/** Returns all descriptors in stable model-identifier order. */
public List<StemmerModelDescriptor> models() { return List.copyOf(this.descriptors.values()); }
/**
* Returns the model with an exact stable identifier.
*
* @param modelId exact stable model identifier
* @return matching descriptor
* @throws NullPointerException if {@code modelId} is {@code null}
* @throws StemmerModelNotFoundException if the selected class loader exposes no matching descriptor
*/
public StemmerModelDescriptor require(final String modelId) {
Objects.requireNonNull(modelId, "modelId");
final StemmerModelDescriptor descriptor = this.descriptors.get(modelId);
if (descriptor == null) {
throw new StemmerModelNotFoundException("No model '" + modelId + "' is available. Add org.egothor:radixor-model-"
+ modelId + ":<version> to the runtime classpath.");
}
return descriptor;
}
/**
* Returns all models for a language in stable model-identifier order.
*
* @param language language to filter
* @return immutable list, possibly empty
* @throws NullPointerException if {@code language} is {@code null}
*/
public List<StemmerModelDescriptor> findByLanguage(final StemmerPatchTrieLoader.Language language) {
Objects.requireNonNull(language, "language");
return this.descriptors.values().stream().filter(value -> value.language() == language).toList();
}
/**
* Resolves the language's documented default model without classpath-order fallback.
*
* @param language language whose {@link StemmerPatchTrieLoader.Language#defaultModelId()} is required
* @return exact default-model descriptor
* @throws NullPointerException if {@code language} is {@code null}
* @throws StemmerModelNotFoundException if the default model is not visible
* @throws StemmerModelIntegrityException if the default descriptor declares another language
*/
public StemmerModelDescriptor requireDefault(final StemmerPatchTrieLoader.Language language) {
Objects.requireNonNull(language, "language");
final StemmerModelDescriptor descriptor = this.descriptors.get(language.defaultModelId());
if (descriptor == null) {
throw new StemmerModelNotFoundException("No default model '" + language.defaultModelId()
+ "' is available for language " + language + ". Add org.egothor:radixor-model-"
+ language.defaultModelId() + ":<version> to the runtime classpath.");
}
if (descriptor.language() != language) {
throw new StemmerModelIntegrityException("Default model '" + descriptor.id() + "' declares language "
+ descriptor.language() + " instead of " + language + ".");
}
return descriptor;
}
/** Reads one deterministic index and appends its descriptors. */
private static void readIndex(final URL index, final ClassLoader classLoader,
final List<StemmerModelDescriptor> descriptors) throws IOException {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(index.openStream(), StandardCharsets.UTF_8))) {
String line;
int lineNumber = 0;
while ((line = reader.readLine()) != null) {
lineNumber++;
final String path = line.trim();
if (path.isEmpty() || path.startsWith("#")) continue;
if (!path.matches("META-INF/radixor/models/[a-z0-9-]+\\.properties")) {
throw new StemmerModelIntegrityException("Malformed model index entry at " + index + ":" + lineNumber + ": " + path);
}
final Enumeration<URL> resources = classLoader.getResources(path);
if (!resources.hasMoreElements()) throw new StemmerModelIntegrityException("Indexed descriptor is missing: " + path + " from " + index);
while (resources.hasMoreElements()) descriptors.add(readDescriptor(resources.nextElement(), classLoader));
}
}
}
/** Parses and validates one immutable descriptor. */
private static StemmerModelDescriptor readDescriptor(final URL source, final ClassLoader classLoader) throws IOException {
final Properties properties = new Properties();
try (InputStream input = source.openStream()) { properties.load(input); }
final String id = required(properties, "model.id", source);
if (!id.matches("[a-z]{2}(?:-[a-z]{2})?-[a-z0-9]+(?:-[a-z0-9]+)*")) throw new StemmerModelIntegrityException("Invalid model.id '" + id + "' at " + source);
final String format = required(properties, "model.format", source);
final int version;
try { version = Integer.parseInt(required(properties, "model.formatVersion", source)); }
catch (NumberFormatException exception) { throw new StemmerModelIntegrityException("Invalid model.formatVersion at " + source, exception); }
if (!FORMAT.equals(format) || version != FORMAT_VERSION) throw new UnsupportedStemmerModelFormatException("Unsupported model format " + format + " version " + version + " at " + source + ".");
final StemmerPatchTrieLoader.Language language;
try { language = StemmerPatchTrieLoader.Language.valueOf(required(properties, "model.language", source)); }
catch (IllegalArgumentException exception) { throw new StemmerModelIntegrityException("Invalid model.language at " + source, exception); }
final String resource = required(properties, "model.resource", source);
if (!resource.equals("org/egothor/stemmer/models/" + id + "/stemmer.gz")) throw new StemmerModelIntegrityException("Invalid model.resource for '" + id + "' at " + source);
if (classLoader.getResource(resource) == null) throw new StemmerModelIntegrityException("Model resource is missing: " + resource + " declared at " + source);
final String checksum = required(properties, "model.sha256", source);
if (!checksum.matches("[0-9a-f]{64}")) throw new StemmerModelIntegrityException("Invalid model.sha256 at " + source);
return new StemmerModelDescriptor(id, required(properties, "model.version", source), language,
required(properties, "model.displayName", source), resource,
Boolean.parseBoolean(required(properties, "model.default", source)), format, version, checksum, source, classLoader);
}
/** Returns a required nonblank property. */
private static String required(final Properties properties, final String key, final URL source) {
final String value = properties.getProperty(key);
if (value == null || value.isBlank()) throw new StemmerModelIntegrityException("Required property '" + key + "' is missing at " + source);
return value.trim();
}
}

10
src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

251
src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -39,6 +39,7 @@ import java.io.PushbackInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@@ -94,13 +95,14 @@ public final class StemmerPatchTrieLoader {
}
/**
* Supported bundled stemmer dictionaries.
* Supported language identities and their stable default model mappings.
*
* <p>
* Each language constant defines:
* </p>
* <ul>
* <li>the resource directory name used under the bundled resources tree</li>
* <li>a deprecated legacy resource-directory name</li>
* <li>the stable default model ID used by language-oriented loading</li>
* <li>whether the language is written right-to-left</li>
* </ul>
*
@@ -115,108 +117,111 @@ public final class StemmerPatchTrieLoader {
/**
* Czech.
*/
CS_CZ("cs_cz", false),
CS_CZ("cs_cz", "cs-cz-default", false),
/**
* Danish.
*/
DA_DK("da_dk", false),
DA_DK("da_dk", "da-dk-default", false),
/**
* German.
*/
DE_DE("de_de", false),
DE_DE("de_de", "de-de-default", false),
/**
* Spanish.
*/
ES_ES("es_es", false),
ES_ES("es_es", "es-es-default", false),
/**
* Persian.
*/
FA_IR("fa_ir", true),
FA_IR("fa_ir", "fa-ir-default", true),
/**
* Finnish.
*/
FI_FI("fi_fi", false),
FI_FI("fi_fi", "fi-fi-default", false),
/**
* French.
*/
FR_FR("fr_fr", false),
FR_FR("fr_fr", "fr-fr-default", false),
/**
* Hebrew.
*/
HE_IL("he_il", true),
HE_IL("he_il", "he-il-default", true),
/**
* Hungarian.
*/
HU_HU("hu_hu", false),
HU_HU("hu_hu", "hu-hu-default", false),
/**
* Italian.
*/
IT_IT("it_it", false),
IT_IT("it_it", "it-it-default", false),
/**
* Norwegian Bokmål.
*/
NB_NO("nb_no", false),
NB_NO("nb_no", "nb-no-default", false),
/**
* Dutch.
*/
NL_NL("nl_nl", false),
NL_NL("nl_nl", "nl-nl-default", false),
/**
* Norwegian Nynorsk.
*/
NN_NO("nn_no", false),
NN_NO("nn_no", "nn-no-default", false),
/**
* Polish.
*/
PL_PL("pl_pl", false),
PL_PL("pl_pl", "pl-pl-unimorph", false),
/**
* Portuguese.
*/
PT_PT("pt_pt", false),
PT_PT("pt_pt", "pt-pt-default", false),
/**
* Russian.
*/
RU_RU("ru_ru", false),
RU_RU("ru_ru", "ru-ru-default", false),
/**
* Swedish.
*/
SV_SE("sv_se", false),
SV_SE("sv_se", "sv-se-default", false),
/**
* Ukrainian.
*/
UK_UA("uk_ua", false),
UK_UA("uk_ua", "uk-ua-default", false),
/**
* English.
*/
US_UK("us_uk", false),
US_UK("us_uk", "us-uk-default", false),
/**
* Yiddish.
*/
YI("yi", true);
YI("yi", "yi-default", true);
/**
* Resource directory name.
*/
private final String resourceDirectory;
/** Stable identifier of the documented default model. */
private final String defaultModelId;
/**
* Whether the language is written right-to-left.
*/
@@ -225,21 +230,37 @@ public final class StemmerPatchTrieLoader {
/**
* Creates a language constant.
*
* @param resourceDirectory resource directory name
* @param resourceDirectory deprecated legacy resource directory name
* @param defaultModelId stable default model identifier
* @param rightToLeft whether the language is written right-to-left
*/
Language(final String resourceDirectory, final boolean rightToLeft) {
Language(final String resourceDirectory, final String defaultModelId, final boolean rightToLeft) {
this.resourceDirectory = resourceDirectory;
this.defaultModelId = defaultModelId;
this.rightToLeft = rightToLeft;
}
/**
* Returns the classpath resource path of the bundled stemmer dictionary.
* Returns the conventional resource path of this language's default model.
*
* <p>Production loading resolves descriptors through
* {@link StemmerModelRegistry}; it does not use this method for discovery or
* selection.</p>
*
* @return classpath resource path
*/
@Deprecated(since = "4.0.0", forRemoval = false)
public String resourcePath() {
return this.resourceDirectory + "/stemmer.gz";
return "org/egothor/stemmer/models/" + this.defaultModelId + "/stemmer.gz";
}
/**
* Returns the stable identifier selected by language-oriented loader methods.
*
* @return exact model ID, independent of classpath ordering
*/
public String defaultModelId() {
return this.defaultModelId;
}
/**
@@ -269,7 +290,7 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a bundled dictionary using explicit reduction settings.
* Loads the language's registered default model using explicit reduction settings.
*
* <p>
* This overload applies the following implicit compilation defaults in addition
@@ -289,7 +310,7 @@ public final class StemmerPatchTrieLoader {
* resulting trie.
* </p>
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionSettings reduction settings
@@ -312,7 +333,7 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a bundled dictionary and returns a runtime-specialized trie whose
* Loads the language's registered default model and returns a runtime-specialized trie whose
* values are compiled patch commands.
*
* <p>
@@ -322,7 +343,7 @@ public final class StemmerPatchTrieLoader {
* runtime stemming does not parse patch-command strings.
* </p>
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionSettings reduction settings
@@ -336,7 +357,7 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a bundled dictionary using explicit trie compilation metadata.
* Loads the language's registered default model using explicit trie compilation metadata.
*
* <p>
* All semantic compilation settings (reduction mode and thresholds, traversal
@@ -345,7 +366,7 @@ public final class StemmerPatchTrieLoader {
* resulting trie.
* </p>
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param metadata trie metadata describing the compilation configuration
@@ -362,20 +383,117 @@ public final class StemmerPatchTrieLoader {
Objects.requireNonNull(language, "language");
Objects.requireNonNull(metadata, "metadata");
final String resourcePath = language.resourcePath();
final StemmerModelDescriptor descriptor = StemmerModelRegistry.fromContextClassLoader().requireDefault(language);
return load(descriptor, storeOriginal, metadata);
}
try (InputStream inputStream = openBundledResource(resourcePath);
/**
* Loads an explicitly selected descriptor using trie compilation metadata.
*
* <p>The method opens the descriptor's namespaced resource through its
* discovering class loader, verifies SHA-256 over the compressed bytes,
* decompresses the GZip UTF-8 dictionary, parses it, and builds a trie. Each
* invocation performs this work and returns serialized patch-command values.</p>
*
* @param descriptor exact validated model descriptor
* @param storeOriginal whether canonical stems receive no-op mappings
* @param metadata trie compilation configuration
* @return newly built trie containing serialized patch-command strings
* @throws NullPointerException if {@code descriptor} or {@code metadata} is {@code null}
* @throws IOException if the compressed dictionary cannot be read or decompressed
* @throws StemmerModelIntegrityException if the resource is missing or its checksum differs
*/
public static FrequencyTrie<String> load(final StemmerModelDescriptor descriptor, final boolean storeOriginal,
final TrieMetadata metadata) throws IOException {
Objects.requireNonNull(descriptor, "descriptor");
Objects.requireNonNull(metadata, "metadata");
try (InputStream inputStream = openModelResource(descriptor);
BufferedReader reader = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
return load(reader, resourcePath, storeOriginal, metadata);
return load(reader, descriptor.resource(), storeOriginal, metadata);
}
}
/**
* Loads a bundled dictionary using explicit trie compilation metadata and
* Loads one exact model descriptor and returns a runtime-specialized trie.
*
* <p>The descriptor's compressed dictionary is integrity-checked, fully
* parsed, reduced with the supplied mode, and converted to immutable
* {@link CompiledPatchCommand} values. The method does not consult a language
* default and does not cache the constructed trie. Applications should retain
* the result for their intended runtime scope. Constructing exceptionally
* large models can require substantial temporary heap.</p>
*
* @param descriptor exact validated model descriptor
* @param storeOriginal whether canonical stems receive no-op mappings
* @param reductionMode reduction mode applied during trie construction
* @return newly constructed trie containing compiled patch commands
* @throws NullPointerException if {@code descriptor} or {@code reductionMode} is {@code null}
* @throws IOException if the compressed dictionary cannot be read or decompressed
* @throws StemmerModelIntegrityException if the resource is missing or its checksum differs
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final StemmerModelDescriptor descriptor,
final boolean storeOriginal, final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(descriptor, "descriptor");
Objects.requireNonNull(reductionMode, "reductionMode");
final TrieMetadata metadata = metadataForCompilation(traversalDirectionOf(descriptor.language()),
ReductionSettings.withDefaults(reductionMode), CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
DiacriticProcessingMode.AS_IS);
return compilePatchTrie(load(descriptor, storeOriginal, metadata));
}
/**
* Loads an exact model ID through a newly discovered context-class-loader registry.
*
* <p>This method is distinct from {@code load(String, ...)}, whose string is a
* filesystem path. Selection is exact and never falls back to another model for
* the same language.</p>
*
* @param modelId exact stable model identifier
* @param storeOriginal whether canonical stems receive no-op mappings
* @param metadata trie compilation configuration
* @return newly built trie containing serialized patch-command strings
* @throws NullPointerException if {@code modelId} or {@code metadata} is {@code null}
* @throws IOException if discovery or resource reading fails
* @throws StemmerModelNotFoundException if the model is not visible
* @throws DuplicateStemmerModelException if discovery finds duplicate IDs
* @throws UnsupportedStemmerModelFormatException if discovery finds an unsupported format
* @throws StemmerModelIntegrityException if metadata or resource integrity is invalid
*/
public static FrequencyTrie<String> loadModel(final String modelId, final boolean storeOriginal,
final TrieMetadata metadata) throws IOException {
return load(StemmerModelRegistry.fromContextClassLoader().require(modelId), storeOriginal, metadata);
}
/** Opens, integrity-checks, and decompresses a descriptor-backed dictionary. */
@SuppressWarnings("PMD.CloseResource")
private static InputStream openModelResource(final StemmerModelDescriptor descriptor) throws IOException {
final InputStream unresolvedResource = descriptor.classLoader().getResourceAsStream(descriptor.resource());
if (unresolvedResource == null) {
throw new StemmerModelIntegrityException("Model resource is missing: " + descriptor.resource());
}
final byte[] bytes;
try (InputStream resource = unresolvedResource) {
bytes = resource.readAllBytes();
}
final String checksum;
try {
checksum = java.util.HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
} catch (java.security.NoSuchAlgorithmException exception) {
throw new StemmerModelIntegrityException("The required SHA-256 algorithm is unavailable.", exception);
}
if (!checksum.equals(descriptor.sha256())) {
throw new StemmerModelIntegrityException("Checksum mismatch for model '" + descriptor.id() + "' at "
+ descriptor.resource() + ": expected " + descriptor.sha256() + " but found " + checksum + ".");
}
return new GZIPInputStream(new java.io.ByteArrayInputStream(bytes));
}
/**
* Loads the language's registered default model using explicit trie compilation metadata and
* returns a runtime-specialized trie whose values are compiled patch commands.
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param metadata trie metadata describing the compilation configuration
@@ -389,7 +507,7 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a bundled dictionary using default settings for the supplied reduction
* Loads the language's registered default model using settings for the supplied reduction
* mode.
*
* <p>
@@ -400,7 +518,7 @@ public final class StemmerPatchTrieLoader {
* diacritic processing mode.
* </p>
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
@@ -419,11 +537,11 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a bundled dictionary using default settings for the supplied reduction
* Loads the language's registered default model using settings for the supplied reduction
* mode and returns a runtime-specialized trie whose values are compiled patch
* commands.
*
* @param language bundled language dictionary
* @param language language whose stable default model is required
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
@@ -1036,9 +1154,11 @@ public final class StemmerPatchTrieLoader {
* @return compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @deprecated Since 2.3.0 for runtime stemming. Use
* {@link #loadCompiled(String, boolean, ReductionMode)} so patch
* commands are represented as {@link CompiledPatchCommand} values.
* @deprecated Since 2.3.0 for runtime stemming. Convert {@code fileName} to a
* {@link Path} and use {@link #loadCompiled(Path, boolean, ReductionMode)}
* so patch commands are represented as {@link CompiledPatchCommand}
* values. The corresponding compiled {@code String} signature is
* reserved for stable model identifiers.
*/
@Deprecated(since = "2.3.0", forRemoval = false)
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
@@ -1048,22 +1168,35 @@ public final class StemmerPatchTrieLoader {
}
/**
* Loads a dictionary from a filesystem path string using default settings for
* the supplied reduction mode and returns runtime-specialized compiled patch
* values.
* Loads one exact stable model identifier and returns a runtime-specialized trie.
*
* @param fileName file name or path string
* <p>The registry is discovered through the thread context class loader.
* Selection is exact: the method never resolves a language default, falls back
* to another model, or depends on classpath order. Use
* {@link #loadCompiled(Path, boolean, ReductionMode)} for a filesystem path.</p>
*
* @param modelId exact stable model identifier
* @param storeOriginal whether the stem itself should be inserted using the
* canonical no-op patch command
* @param reductionMode reduction mode
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if the file cannot be opened or read
* @throws IllegalArgumentException if {@code modelId} is blank
* @throws IOException if registry discovery or dictionary reading fails
* @throws StemmerModelNotFoundException if the exact model is not visible
* @throws DuplicateStemmerModelException if discovery finds duplicate model IDs
* @throws UnsupportedStemmerModelFormatException if a descriptor format is unsupported
* @throws StemmerModelIntegrityException if descriptor or resource integrity validation fails
*/
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String fileName,
public static FrequencyTrie<CompiledPatchCommand> loadCompiled(final String modelId,
final boolean storeOriginal, final ReductionMode reductionMode) throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return loadCompiled(Path.of(fileName), storeOriginal, reductionMode);
Objects.requireNonNull(modelId, "modelId");
Objects.requireNonNull(reductionMode, "reductionMode");
if (modelId.isBlank()) {
throw new IllegalArgumentException("modelId must not be blank");
}
final StemmerModelDescriptor descriptor = StemmerModelRegistry.fromContextClassLoader().require(modelId);
return loadCompiled(descriptor, storeOriginal, reductionMode);
}
/**
@@ -1125,9 +1258,9 @@ public final class StemmerPatchTrieLoader {
}
/**
* Resolves the traversal direction implied by a bundled language definition.
* Resolves the traversal direction implied by a language definition.
*
* @param language bundled language
* @param language language definition
* @return traversal direction to use for that language
*/
private static WordTraversalDirection traversalDirectionOf(final Language language) {

10
src/main/java/org/egothor/stemmer/TrieMetadata.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

46
src/main/java/org/egothor/stemmer/UnsupportedStemmerModelFormatException.java Normal file
View File

@@ -0,0 +1,46 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
/**
* Indicates that a descriptor declares a model format name or format version not
* supported by the current Radixor core.
*/
public final class UnsupportedStemmerModelFormatException extends IllegalStateException {
private static final long serialVersionUID = 1L;
/**
* Creates an exception describing the unsupported format.
* @param message diagnostic naming the format, version, and descriptor source
*/
public UnsupportedStemmerModelFormatException(final String message) {
super(message);
}
}

10
src/main/java/org/egothor/stemmer/ValueCount.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/WordTraversalDirection.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/package-info.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/ChildDescriptor.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/DominantLocalDescriptor.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/LocalValueSummary.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/MutableNode.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/NodeData.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/RankedLocalDescriptor.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/ReducedNode.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/ReductionContext.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/ReductionSignature.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/SortableValue.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/UnorderedLocalDescriptor.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/main/java/org/egothor/stemmer/trie/package-info.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

BIN
src/main/resources/cs_cz/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/da_dk/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/de_de/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/es_es/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/fa_ir/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/fi_fi/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/fr_fr/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/he_il/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/hu_hu/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/it_it/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/nb_no/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/nl_nl/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/nn_no/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/pl_pl/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/pt_pt/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/ru_ru/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/sv_se/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/uk_ua/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/us_uk/stemmer.gz
View File

Binary file not shown.

BIN
src/main/resources/yi/stemmer.gz
View File

Binary file not shown.

18
src/test/java/org/egothor/stemmer/CompileIntegrationTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -334,7 +334,7 @@ final class CompileIntegrationTest {
* </p>
*
* @param scenario scenario identifier
* @param resourcePath bundled dictionary resource path
* @param resourcePath registered model dictionary resource path
* @throws IOException if reading or writing fails
*/
@ParameterizedTest(name = "[{index}] {0}")
@@ -359,7 +359,7 @@ final class CompileIntegrationTest {
final Map<String, Set<String>> representativeStemsByVariant = readRepresentativeVariantExpectations(
resourcePath, REPRESENTATIVE_VARIANT_LIMIT);
assertFalse(representativeStemsByVariant.isEmpty(), "The bundled dictionary must provide at least one "
assertFalse(representativeStemsByVariant.isEmpty(), "The registered model dictionary must provide at least one "
+ "representative variant without Unicode whitespace for " + scenario + '.');
for (Map.Entry<String, Set<String>> entry : representativeStemsByVariant.entrySet()) {
@@ -451,7 +451,7 @@ final class CompileIntegrationTest {
* </p>
*
* <p>
* The bundled dictionary format is expected to be tab-separated values, meaning
* The registered model dictionary format is expected to be tab-separated values, meaning
* that columns are separated by the tab character:
* </p>
*
@@ -467,7 +467,7 @@ final class CompileIntegrationTest {
* helper.
* </p>
*
* @param resourcePath bundled dictionary resource path
* @param resourcePath registered model dictionary resource path
* @param limit maximum number of representative variants to collect
* @return representative variants mapped to their acceptable stems
* @throws IOException if reading fails

10
src/test/java/org/egothor/stemmer/CompileTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/DiacriticStripperTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/FrequencyTrieBuildersTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/FrequencyTrieProperties.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

137
src/test/java/org/egothor/stemmer/FullRuntimeModelIntegrationTest.java Normal file
View File

@@ -0,0 +1,137 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/**
* Constructs and exercises a complete packaged model selected by a Gradle
* property in a dedicated memory-sized test process.
*/
@Tag("large-model")
final class FullRuntimeModelIntegrationTest {
/** System property containing the exact model identifier under verification. */
private static final String MODEL_ID_PROPERTY = "radixor.test.modelId";
/** Reduction mode used by the supported runtime construction path. */
private static final ReductionMode REDUCTION_MODE =
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS;
/**
* Loads the entire selected resource, constructs its compiled trie, and applies
* deterministic PoliMorf smoke fixtures when that model is selected.
*
* @throws IOException if discovery or complete dictionary processing fails
*/
@Test
@DisplayName("Complete packaged model constructs a compiled runtime trie")
void constructsCompletePackagedRuntimeModel() throws IOException {
final String modelId = requiredModelId();
final long startedAt = System.nanoTime();
final StemmerModelRegistry registry = StemmerModelRegistry.fromContextClassLoader();
final StemmerModelDescriptor descriptor = registry.require(modelId);
assertNotNull(descriptor.source());
assertNotNull(descriptor.classLoader().getResource(descriptor.resource()));
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(
modelId, true, REDUCTION_MODE);
final long elapsedNanos = System.nanoTime() - startedAt;
assertTrue(trie.size() > 0, "The completely constructed trie must contain canonical nodes.");
if ("pl-pl-polimorf".equals(modelId)) {
verifyPolimorfFixtures(trie);
}
System.out.printf(
"RUNTIME_MODEL_METRICS model=%s maxHeapBytes=%d elapsedMillis=%d canonicalNodes=%d%n",
modelId, Runtime.getRuntime().maxMemory(), elapsedNanos / 1_000_000L, trie.size());
}
/**
* Returns the nonblank exact model identifier supplied by the Gradle task.
*
* @return exact model identifier
*/
private static String requiredModelId() {
final String modelId = System.getProperty(MODEL_ID_PROPERTY);
if (modelId == null || modelId.isBlank()) {
throw new IllegalStateException("System property " + MODEL_ID_PROPERTY + " must name a model.");
}
return modelId;
}
/**
* Verifies stable forms selected from the immutable PoliMorf module input.
* The first column of each source row is the expected lemma and subsequent
* columns contain its forms.
*
* @param trie completely constructed PoliMorf trie
*/
private static void verifyPolimorfFixtures(final FrequencyTrie<CompiledPatchCommand> trie) {
assertStem(trie, "pies", "pies");
assertStem(trie, "psami", "pies");
assertStem(trie, "kotem", "kot");
assertStem(trie, "zamkami", "zamek");
final List<String> candidates = Arrays.stream(trie.getAll("mam"))
.map(command -> command.apply("mam"))
.toList();
assertArrayEquals(new String[]{"mama", "mamić", "mieć"}, candidates.toArray(String[]::new));
assertNull(trie.get("radixorbrakujacehaslo"));
assertEquals(0, trie.getAll("radixorbrakujacehaslo").length);
}
/**
* Applies the preferred compiled patch command and compares its result with a
* reviewed source-dictionary lemma.
*
* @param trie PoliMorf trie
* @param form inflected or lemma form
* @param expectedLemma expected source-dictionary lemma
*/
private static void assertStem(final FrequencyTrie<CompiledPatchCommand> trie, final String form,
final String expectedLemma) {
final CompiledPatchCommand command = trie.get(form);
assertNotNull(command, "A reviewed PoliMorf form must have a patch command: " + form);
assertEquals(expectedLemma, command.apply(form));
}
}

10
src/test/java/org/egothor/stemmer/FuzzStemmerAndTrieCompilationTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/FuzzTestSupport.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

195
src/test/java/org/egothor/stemmer/ModelDependencyResolutionTest.java Normal file
View File

@@ -0,0 +1,195 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import org.gradle.testkit.runner.BuildResult;
import org.gradle.testkit.runner.GradleRunner;
import org.gradle.testkit.runner.TaskOutcome;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
/**
* Verifies consumer-visible dependency resolution from the isolated local Maven repository.
*/
@Tag("integration")
@DisplayName("Published model dependency topology")
class ModelDependencyResolutionTest {
/** The temporary directory used for isolated Gradle consumer builds. */
@TempDir
Path temporaryDirectory;
/**
* Verifies that the standard aggregate resolves all defaults and excludes optional PoliMorf.
*
* @throws IOException if the consumer fixture cannot be created or read
*/
@Test
@DisplayName("Standard aggregate resolves twenty defaults and excludes PoliMorf")
void standardAggregateResolvesDefaultsOnly() throws IOException {
Set<String> artifacts = resolve("""
implementation 'org.egothor:radixor:%s'
runtimeOnly 'org.egothor:radixor-models-standard:%s'
""".formatted(coreVersion(), catalogVersion()));
assertTrue(artifacts.contains("radixor"));
assertTrue(artifacts.contains("radixor-model-pl-pl-unimorph"));
assertFalse(artifacts.contains("radixor-model-pl-pl-polimorf"));
assertEquals(21, artifacts.size(), "The core and twenty default model JARs must resolve.");
}
/**
* Verifies that importing the model BOM alone introduces no model artifact.
*
* @throws IOException if the consumer fixture cannot be created or read
*/
@Test
@DisplayName("BOM alone contributes constraints but no model JAR")
void bomAloneIntroducesNoModels() throws IOException {
Set<String> artifacts = resolve("implementation platform('org.egothor:radixor-models-bom:"
+ catalogVersion() + "')");
assertTrue(artifacts.isEmpty(), "A dependency-management BOM must not add runtime artifacts.");
}
/**
* Verifies that the BOM supplies the source-controlled PoliMorf model version.
*
* @throws IOException if the consumer fixture cannot be created or read
*/
@Test
@DisplayName("BOM manages an explicitly requested PoliMorf model")
void bomManagesExplicitPolimorf() throws IOException {
Set<String> artifacts = resolve("""
implementation platform('org.egothor:radixor-models-bom:%s')
runtimeOnly 'org.egothor:radixor-model-pl-pl-polimorf'
""".formatted(catalogVersion()));
assertEquals(Set.of("radixor-model-pl-pl-polimorf"), artifacts);
}
/**
* Verifies that the root core publication has no transitive model dependency.
*
* @throws IOException if the consumer fixture cannot be created or read
*/
@Test
@DisplayName("Core publication resolves without model artifacts")
void coreHasNoTransitiveModels() throws IOException {
Set<String> artifacts = resolve("implementation 'org.egothor:radixor:" + coreVersion() + "'");
assertEquals(Set.of("radixor"), artifacts);
}
/**
* Executes an isolated offline Gradle consumer build and returns resolved artifact identifiers.
*
* @param dependencyDeclarations Gradle dependency declarations for the fixture
* @return deterministically ordered resolved Maven artifact identifiers
* @throws IOException if fixture files cannot be created or read
*/
private Set<String> resolve(final String dependencyDeclarations) throws IOException {
Path fixtureDirectory = Files.createTempDirectory(temporaryDirectory, "consumer-");
Path repository = Path.of(requiredProperty("radixor.consumer.repository"));
Files.writeString(fixtureDirectory.resolve("settings.gradle"), "rootProject.name = 'consumer'\n",
StandardCharsets.UTF_8);
Files.writeString(fixtureDirectory.resolve("build.gradle"), """
plugins { id 'java' }
repositories { maven { url = uri('%s') } }
dependencies {
%s
}
tasks.register('resolveRuntime') {
doLast {
def names = configurations.runtimeClasspath.resolvedConfiguration.resolvedArtifacts
.collect { it.moduleVersion.id.name }.toSorted()
file('resolved.txt').text = names.join('\\n') + (names.isEmpty() ? '' : '\\n')
}
}
""".formatted(repository.toUri(), dependencyDeclarations), StandardCharsets.UTF_8);
BuildResult result = GradleRunner.create()
.withProjectDir(fixtureDirectory.toFile())
.withArguments("--offline", "--stacktrace", "resolveRuntime")
.build();
assertEquals(TaskOutcome.SUCCESS, result.task(":resolveRuntime").getOutcome());
Path resultFile = fixtureDirectory.resolve("resolved.txt");
List<String> lines = Files.exists(resultFile)
? Files.readAllLines(resultFile, StandardCharsets.UTF_8)
: List.of();
return new TreeSet<>(lines);
}
/**
* Returns the root core version supplied by the Gradle test task.
*
* @return current root core version
*/
private static String coreVersion() {
return requiredProperty("radixor.core.version");
}
/**
* Returns the model catalog version supplied by the Gradle test task.
*
* @return current model catalog version
*/
private static String catalogVersion() {
return requiredProperty("radixor.catalog.version");
}
/**
* Returns a required system property or fails with an actionable diagnostic.
*
* @param name system-property name
* @return nonblank property value
*/
private static String requiredProperty(final String name) {
String value = System.getProperty(name);
if (value == null || value.isBlank()) {
throw new IllegalStateException("Required test system property is missing: " + name);
}
return value;
}
}

10
src/test/java/org/egothor/stemmer/PatchCommandEncoderProperties.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/PropertyBasedTestSupport.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/RegressionArtifactGenerator.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/RegressionArtifactSupport.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/StemmerDictionaryParserTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/StemmerKnowledgeExperimentTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

174
src/test/java/org/egothor/stemmer/StemmerModelDocumentationExamplesTest.java Normal file
View File

@@ -0,0 +1,174 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.List;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/** Executes the essential model discovery and loading examples used by maintained documentation. */
@Tag("documentation")
@Tag("integration")
final class StemmerModelDocumentationExamplesTest {
/** Registry discovered from the standard test runtime classpath. */
private static StemmerModelRegistry registry;
/** Discovers the documented models once for this example suite. */
@BeforeAll
static void discoverDocumentedModels() throws IOException {
registry = StemmerModelRegistry.fromContextClassLoader();
}
/** Verifies that language-oriented loading retains the documented Polish default. */
@Test
@DisplayName("Documentation example loads the default Polish UniMorph model")
void loadsDefaultPolishModel() throws IOException {
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.PL_PL,
true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
final CompiledPatchCommand command = trie.get("koty");
assertNotNull(command);
assertEquals("kot", command.apply("koty"));
assertEquals("pl-pl-unimorph",
registry.requireDefault(StemmerPatchTrieLoader.Language.PL_PL).id());
}
/** Verifies exact PoliMorf descriptor selection and its packaged runtime metadata. */
@Test
@DisplayName("Documentation example selects PoliMorf by exact model ID")
void selectsExplicitPolimorfModel() {
final StemmerModelDescriptor descriptor = registry.require("pl-pl-polimorf");
assertEquals("PL_PL", descriptor.language().name());
assertEquals("org/egothor/stemmer/models/pl-pl-polimorf/stemmer.gz", descriptor.resource());
assertEquals("1.0.0", descriptor.version());
}
/** Verifies the documented explicit descriptor-to-compiled-trie API with a registered model. */
@Test
@DisplayName("Documentation example loads an exact descriptor into a usable trie")
void loadsExplicitDescriptor() throws IOException {
final StemmerModelDescriptor descriptor = registry.require("pl-pl-unimorph");
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(
descriptor, true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
final CompiledPatchCommand command = trie.get("koty");
assertEquals("kot", command.apply("koty"));
}
/** Verifies exact model-ID compiled loading without language-default fallback. */
@Test
@DisplayName("Documentation example loads an exact model ID into a compiled trie")
void loadsExplicitModelId() throws IOException {
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(
"pl-pl-unimorph", true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
assertEquals("kot", trie.get("koty").apply("koty"));
}
/** Verifies null, blank, and unknown exact model-ID failure behavior. */
@Test
@DisplayName("Documentation example rejects invalid or unavailable explicit model IDs")
void rejectsInvalidExplicitModelIds() {
assertThrows(NullPointerException.class, () -> StemmerPatchTrieLoader.loadCompiled(
(String) null, true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
assertThrows(NullPointerException.class, () -> StemmerPatchTrieLoader.loadCompiled(
(StemmerModelDescriptor) null, true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
assertThrows(NullPointerException.class, () -> StemmerPatchTrieLoader.loadCompiled(
registry.require("pl-pl-unimorph"), true, (ReductionMode) null));
assertThrows(IllegalArgumentException.class, () -> StemmerPatchTrieLoader.loadCompiled(
" ", true, ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
assertThrows(StemmerModelNotFoundException.class, () -> StemmerPatchTrieLoader.loadCompiled(
"pl-pl-does-not-exist", true,
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
}
/** Verifies that two models coexist without changing default resolution. */
@Test
@DisplayName("Documentation example keeps both Polish models independent")
void loadsBothPolishModelsIndependently() {
final StemmerModelDescriptor unimorph = registry.require("pl-pl-unimorph");
final StemmerModelDescriptor polimorf = registry.require("pl-pl-polimorf");
final StemmerModelDescriptor defaultModel = registry.requireDefault(StemmerPatchTrieLoader.Language.PL_PL);
assertEquals("pl-pl-unimorph", defaultModel.id());
assertEquals(StemmerPatchTrieLoader.Language.PL_PL, unimorph.language());
assertEquals(StemmerPatchTrieLoader.Language.PL_PL, polimorf.language());
assertFalse(unimorph.id().equals(polimorf.id()));
}
/** Verifies deterministic discovery and language filtering shown in documentation. */
@Test
@DisplayName("Documentation example lists and filters models deterministically")
void discoversAndFiltersModels() {
final List<StemmerModelDescriptor> polish = registry.findByLanguage(StemmerPatchTrieLoader.Language.PL_PL);
assertEquals(List.of("pl-pl-polimorf", "pl-pl-unimorph"),
polish.stream().map(StemmerModelDescriptor::id).toList());
assertEquals(registry.models().stream().sorted().toList(), registry.models());
assertTrue(polish.stream().allMatch(model -> model.format().equals("radixor-dictionary-tsv-gzip")));
}
/** Verifies discovery through an explicitly supplied application class loader. */
@Test
@DisplayName("Documentation example discovers models through an explicit ClassLoader")
void discoversThroughExplicitClassLoader() throws IOException {
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
final StemmerModelRegistry explicitRegistry = StemmerModelRegistry.fromClassLoader(classLoader);
assertEquals("pl-pl-polimorf", explicitRegistry.require("pl-pl-polimorf").id());
assertEquals(registry.models().stream().map(StemmerModelDescriptor::id).toList(),
explicitRegistry.models().stream().map(StemmerModelDescriptor::id).toList());
}
/** Verifies the exact missing-model failure produced by an isolated empty loader. */
@Test
@DisplayName("Documentation example reports a model missing from an isolated ClassLoader")
void reportsMissingModelFromIsolatedClassLoader() throws IOException {
try (URLClassLoader isolated = new URLClassLoader(new URL[0], null)) {
final StemmerModelRegistry emptyRegistry = StemmerModelRegistry.fromClassLoader(isolated);
final StemmerModelNotFoundException exception = assertThrows(StemmerModelNotFoundException.class,
() -> emptyRegistry.require("pl-pl-polimorf"));
assertTrue(exception.getMessage().contains(
"org.egothor:radixor-model-pl-pl-polimorf:<version>"));
}
}
}

82
src/test/java/org/egothor/stemmer/StemmerModelRegistryTest.java Normal file
View File

@@ -0,0 +1,82 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.List;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
/** Tests deterministic model discovery and language/model separation. */
@Tag("integration")
final class StemmerModelRegistryTest {
/** Verifies discovery, ordering, coexistence, and the Polish default. */
@Test
@DisplayName("Registry discovers all models deterministically and keeps UniMorph as Polish default")
void discoversModelsDeterministically() throws IOException {
final StemmerModelRegistry registry = StemmerModelRegistry.fromContextClassLoader();
final List<StemmerModelDescriptor> polish = registry.findByLanguage(StemmerPatchTrieLoader.Language.PL_PL);
assertEquals(List.of("pl-pl-polimorf", "pl-pl-unimorph"),
polish.stream().map(StemmerModelDescriptor::id).toList());
assertEquals("pl-pl-unimorph", registry.requireDefault(StemmerPatchTrieLoader.Language.PL_PL).id());
assertEquals("pl-pl-polimorf", registry.require("pl-pl-polimorf").id());
assertEquals(registry.models().stream().sorted().toList(), registry.models());
}
/** Verifies that missing explicit models never fall back arbitrarily. */
@Test
@DisplayName("Missing explicit model reports its exact dependency")
void rejectsMissingModel() throws IOException {
final StemmerModelRegistry registry = StemmerModelRegistry.fromContextClassLoader();
final StemmerModelNotFoundException exception = assertThrows(StemmerModelNotFoundException.class,
() -> registry.require("pl-pl-unknown"));
assertTrue(exception.getMessage().contains("org.egothor:radixor-model-pl-pl-unknown:<version>"));
assertFalse(exception.getMessage().isBlank());
}
/** Verifies the source-compatible language loader resolves the registered default. */
@Test
@Tag("slow")
@DisplayName("Polish language loading resolves the UniMorph model")
void loadsDefaultPolishModel() throws IOException {
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader.loadCompiled(
StemmerPatchTrieLoader.Language.PL_PL, true,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
assertTrue(trie.metadata() != null);
}
}

10
src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

29
src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -73,7 +73,7 @@ import org.junit.jupiter.params.provider.MethodSource;
*
* <p>
* The suite combines focused API-level verification with integration validation
* against bundled dictionaries. It verifies:
* against registered default-model dictionaries. It verifies:
* </p>
* <ul>
* <li>all public loading overloads</li>
@@ -113,7 +113,7 @@ final class StemmerPatchTrieLoaderTest {
private static final int REPRESENTATIVE_BUNDLED_WORD_COUNT = 25;
/**
* Provides arguments for bundled dictionary verification across both supported
* Provides arguments for registered model dictionary verification across both supported
* getAll-preserving reduction modes.
*
* <p>
@@ -521,13 +521,10 @@ final class StemmerPatchTrieLoaderTest {
dictionaryFile, true, DEFAULT_REDUCTION_MODE);
final FrequencyTrie<CompiledPatchCommand> fromStringWithSettings = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile.toString(), true, settings);
final FrequencyTrie<CompiledPatchCommand> fromStringWithMode = StemmerPatchTrieLoader.loadCompiled(
dictionaryFile.toString(), true, DEFAULT_REDUCTION_MODE);
assertCompiledTrieSemanticsEqual(expected, fromPathWithSettings, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromPathWithMode, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromStringWithSettings, "running", "played", "cities", "run");
assertCompiledTrieSemanticsEqual(expected, fromStringWithMode, "running", "played", "cities", "run");
}
/**
@@ -821,14 +818,14 @@ final class StemmerPatchTrieLoaderTest {
}
/**
* Verifies that each bundled dictionary compiles into a trie whose
* Verifies that each registered default-model dictionary compiles into a trie whose
* {@link FrequencyTrie#getAll(String)} results still reconstruct exactly the
* same set of stems as the source dictionary.
*
* @param scenario human-readable numbered scenario identifier
* @param language tested bundled language
* @param reductionMode reduction mode
* @throws IOException if a bundled dictionary cannot be read
* @throws IOException if a registered model dictionary cannot be read
*/
@ParameterizedTest(name = "[{index}] {0}")
@MethodSource("org.egothor.stemmer.StemmerPatchTrieLoaderTest#bundledDictionaryCases")
@@ -861,7 +858,7 @@ final class StemmerPatchTrieLoaderTest {
}
/**
* Verifies that representative bundled dictionaries load equivalently through
* Verifies that representative registered model dictionaries load equivalently through
* both reduction-setting and reduction-mode overloads.
*
* @param scenario scenario identifier
@@ -890,12 +887,12 @@ final class StemmerPatchTrieLoaderTest {
}
assertFalse(expectedStemsByWord.isEmpty(),
"Scenario " + scenario + " must provide at least one bundled dictionary entry.");
"Scenario " + scenario + " must provide at least one registered model dictionary entry.");
}
}
/**
* Reads the bundled dictionary and builds a mapping of surface word to all
* Reads the registered model dictionary and builds a mapping of surface word to all
* stems it is associated with in the source data.
*
* <p>
@@ -1034,7 +1031,7 @@ final class StemmerPatchTrieLoaderTest {
}
/**
* Opens one bundled dictionary resource.
* Opens one registered model dictionary resource.
*
* @param resourcePath classpath resource path
* @return opened input stream

24
src/test/java/org/egothor/stemmer/StemmerPatchTrieProperties.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -85,13 +85,15 @@ class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
assertTrue(acceptableStems.contains(PatchCommandEncoder.apply(observedWord, preferredPatch, trie.traversalDirection())),
"preferred patch reconstructed an unexpected stem.");
final Set<String> producedStems = applyAll(trie, observedWord, allPatches);
assertTrue(acceptableStems.containsAll(producedStems),
"getAll() must not expose a patch that reconstructs an undeclared stem.");
if (reductionMode != ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS) {
final Set<String> producedStems = applyAll(trie, observedWord, allPatches);
assertTrue(acceptableStems.containsAll(producedStems),
"A getAll()-preserving mode must not reconstruct an undeclared stem.");
if (acceptableStems.contains(observedWord)) {
assertTrue(producedStems.contains(observedWord),
"storeOriginal semantics must preserve the original stem among returned results.");
if (acceptableStems.contains(observedWord)) {
assertTrue(producedStems.contains(observedWord),
"A getAll()-preserving mode must retain the stored original result.");
}
}
}
}

10
src/test/java/org/egothor/stemmer/TrieMetadataTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/WordTraversalDirectionTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/benchmark/BenchmarkTokenSequenceTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/benchmark/EnglishStemmerComparisonTokenStreamTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

10
src/test/java/org/egothor/stemmer/benchmark/PaiceHuskLancasterStemmerTest.java
View File

@@ -1,21 +1,21 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

35
src/test/java/org/egothor/stemmer/benchmark/quality/BundledGoldStandardLoader.java
View File

@@ -1,3 +1,33 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark.quality;
import java.io.BufferedReader;
@@ -15,7 +45,7 @@ import org.egothor.stemmer.CaseProcessingMode;
import org.egothor.stemmer.StemmerDictionaryParser;
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
/** Loads gold-standard groups from authoritative bundled dictionary resources. */
/** Loads gold-standard groups from authoritative registered model resources. */
public final class BundledGoldStandardLoader {
/** Utility class. */
private BundledGoldStandardLoader() { throw new AssertionError("No instances."); }
@@ -28,7 +58,8 @@ public final class BundledGoldStandardLoader {
*/
public static List<GoldStandardGroup> load(final Language language) throws IOException {
Objects.requireNonNull(language, "language");
final String resource = language.resourcePath();
final String resource = org.egothor.stemmer.StemmerModelRegistry.fromContextClassLoader()
.requireDefault(language).resource();
final List<GoldStandardGroup> groups = new ArrayList<>();
try (InputStream raw = openResource(language, resource); InputStream gzip = new GZIPInputStream(raw);
BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {

Some files were not shown because too many files have changed in this diff Show More