Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(python): add native distribution and release infrastructure

Browse Source 
- add the Rust-backed Python API with PyStemmer compatibility
- distribute standard compiled models as a separate Python package
- generate model artifacts during builds instead of storing them in Git
- add GitHub release and Pages-backed package index workflows
- add Python tests, benchmarks, documentation, and Gradle integration
- refresh the documentation site, branding, and language benchmarks
This commit is contained in:
Leo Galambos
2026-08-10 22:34:32 +02:00
parent b45e143c84
commit 5e3d3c7c7d
139 changed files with 11420 additions and 747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageCase.java
View File

@@ -31,6 +31,7 @@
package org.egothor.stemmer.benchmark;
import org.egothor.stemmer.StemmerPatchTrieLoader;
import org.egothor.stemmer.benchmark.snowball.ext.czechStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.danishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.dutchStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.finnishStemmer;
@@ -39,6 +40,8 @@ import org.egothor.stemmer.benchmark.snowball.ext.germanStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.hungarianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.italianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.norwegianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.persianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.polishStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.portugueseStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.russianStemmer;
import org.egothor.stemmer.benchmark.snowball.ext.spanishStemmer;
@@ -50,6 +53,11 @@ import org.egothor.stemmer.benchmark.snowball.ext.yiddishStemmer;
*/
enum SnowballLanguageCase {
/**
* Czech Snowball stemming over the Radixor Czech dictionary.
*/
CZECH("Czech", StemmerPatchTrieLoader.Language.CS_CZ, czechStemmer::new),
/**
* Danish Snowball stemming over the Radixor Danish dictionary.
*/
@@ -97,6 +105,16 @@ enum SnowballLanguageCase {
NORWEGIAN_NYNORSK("Norwegian Nynorsk", StemmerPatchTrieLoader.Language.NN_NO, norwegianStemmer::new,
"Norwegian"),
/**
* Persian Snowball stemming over the Radixor Persian dictionary.
*/
PERSIAN("Persian", StemmerPatchTrieLoader.Language.FA_IR, persianStemmer::new),
/**
* Polish Snowball stemming over the Radixor Polish dictionary.
*/
POLISH("Polish", StemmerPatchTrieLoader.Language.PL_PL, polishStemmer::new),
/**
* Portuguese Snowball stemming over the Radixor Portuguese dictionary.
*/
@@ -142,6 +160,19 @@ enum SnowballLanguageCase {
*/
private final String luceneSnowballName;
/**
* Creates a direct-only language case not provided by the current Lucene
* Snowball implementation.
*
* @param displayLanguage human-readable language name
* @param radixorLanguage matching Radixor language resource
* @param directFactory direct Snowball stemmer factory
*/
SnowballLanguageCase(final String displayLanguage, final StemmerPatchTrieLoader.Language radixorLanguage,
final SnowballStemmerAdapter.Factory directFactory) {
this(displayLanguage, radixorLanguage, directFactory, null);
}
/**
* Creates a language case.
*
@@ -191,6 +222,9 @@ enum SnowballLanguageCase {
* @return Lucene SnowballFilter algorithm name
*/
String luceneSnowballName() {
if (this.luceneSnowballName == null) {
throw new IllegalStateException("Lucene Snowball does not provide " + this.displayLanguage);
}
return this.luceneSnowballName;
}
}

51
src/jmh/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmark.java
View File

@@ -116,6 +116,49 @@ public class SnowballLanguageStemmerComparisonBenchmark {
}
}
/**
* Shared corpus state for every official direct Snowball implementation.
*
* <p>
* Czech, Persian, and Polish are available in the official Snowball 3.1.0
* distribution but not through the Lucene SnowballFilter version used by
* this project. Keeping the direct parameter domain separate prevents JMH
* from constructing unsupported Lucene workloads.
* </p>
*/
@State(Scope.Benchmark)
public static class DirectSharedState {
/**
* Language/algorithm case under comparison.
*/
@Param({ "CZECH", "DANISH", "DUTCH", "FINNISH", "FRENCH", "GERMAN", "HUNGARIAN", "ITALIAN",
"NORWEGIAN_BOKMAL", "NORWEGIAN_NYNORSK", "PERSIAN", "POLISH", "PORTUGUESE", "RUSSIAN",
"SPANISH", "SWEDISH", "YIDDISH" })
public String languageCaseName;
/**
* Resolved language/algorithm case.
*/
private SnowballLanguageCase languageCase;
/**
* Shared deterministic changed-token dictionary corpus.
*/
private String[] tokens;
/**
* Initializes the selected direct Snowball corpus before measurement.
*
* @throws IOException if the corpus cannot be loaded
*/
@Setup(Level.Trial)
public void setUp() throws IOException {
this.languageCase = SnowballLanguageCase.valueOf(this.languageCaseName);
this.tokens = LanguageBenchmarkCorpus.createTokens(this.languageCase.radixorLanguage());
}
}
/**
* Per-thread direct Snowball state.
*/
@@ -130,10 +173,10 @@ public class SnowballLanguageStemmerComparisonBenchmark {
/**
* Initializes direct Snowball state for the selected language.
*
* @param sharedState selected language state
* @param sharedState selected direct Snowball language state
*/
@Setup(Level.Trial)
public void setUp(final SharedState sharedState) {
public void setUp(final DirectSharedState sharedState) {
this.snowballStemmer = sharedState.languageCase.createDirectStemmer();
}
}
@@ -198,7 +241,7 @@ public class SnowballLanguageStemmerComparisonBenchmark {
/**
* Runs Radixor over the selected Snowball-language corpus.
*
* @param sharedState shared benchmark state
* @param sharedState shared direct Snowball benchmark state
* @param blackhole result sink
*/
@Benchmark
@@ -220,7 +263,7 @@ public class SnowballLanguageStemmerComparisonBenchmark {
* @param blackhole result sink
*/
@Benchmark
public void snowballDirect(final SharedState sharedState, final DirectState directState,
public void snowballDirect(final DirectSharedState sharedState, final DirectState directState,
final Blackhole blackhole) {
final String[] tokens = sharedState.tokens;
final SnowballStemmerAdapter stemmer = directState.snowballStemmer;

6
src/jmh/java/org/egothor/stemmer/benchmark/StemmerComparisonBenchmarkQuality.java
View File

@@ -179,6 +179,7 @@ public class StemmerComparisonBenchmarkQuality {
"UKRAINIAN_RADIXOR",
"UKRAINIAN_MORFOLOGIK_DIRECT",
"UKRAINIAN_LUCENE_MORFOLOGIK_FILTER",
"SNOWBALL_CZECH_DIRECT",
"SNOWBALL_DANISH_DIRECT",
"SNOWBALL_DANISH_LUCENE_FILTER",
"SNOWBALL_DUTCH_DIRECT",
@@ -197,6 +198,8 @@ public class StemmerComparisonBenchmarkQuality {
"SNOWBALL_NORWEGIAN_BOKMAL_LUCENE_FILTER",
"SNOWBALL_NORWEGIAN_NYNORSK_DIRECT",
"SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER",
"SNOWBALL_PERSIAN_DIRECT",
"SNOWBALL_POLISH_DIRECT",
"SNOWBALL_PORTUGUESE_DIRECT",
"SNOWBALL_PORTUGUESE_LUCENE_FILTER",
"SNOWBALL_RUSSIAN_DIRECT",
@@ -365,6 +368,7 @@ public class StemmerComparisonBenchmarkQuality {
UKRAINIAN_RADIXOR(StemmerPatchTrieLoader.Language.UK_UA),
UKRAINIAN_MORFOLOGIK_DIRECT(StemmerPatchTrieLoader.Language.UK_UA),
UKRAINIAN_LUCENE_MORFOLOGIK_FILTER(StemmerPatchTrieLoader.Language.UK_UA),
SNOWBALL_CZECH_DIRECT(StemmerPatchTrieLoader.Language.CS_CZ, SnowballLanguageCase.CZECH),
SNOWBALL_DANISH_DIRECT(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
SNOWBALL_DANISH_LUCENE_FILTER(StemmerPatchTrieLoader.Language.DA_DK, SnowballLanguageCase.DANISH),
SNOWBALL_DUTCH_DIRECT(StemmerPatchTrieLoader.Language.NL_NL, SnowballLanguageCase.DUTCH),
@@ -387,6 +391,8 @@ public class StemmerComparisonBenchmarkQuality {
SnowballLanguageCase.NORWEGIAN_NYNORSK),
SNOWBALL_NORWEGIAN_NYNORSK_LUCENE_FILTER(StemmerPatchTrieLoader.Language.NN_NO,
SnowballLanguageCase.NORWEGIAN_NYNORSK),
SNOWBALL_PERSIAN_DIRECT(StemmerPatchTrieLoader.Language.FA_IR, SnowballLanguageCase.PERSIAN),
SNOWBALL_POLISH_DIRECT(StemmerPatchTrieLoader.Language.PL_PL, SnowballLanguageCase.POLISH),
SNOWBALL_PORTUGUESE_DIRECT(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
SNOWBALL_PORTUGUESE_LUCENE_FILTER(StemmerPatchTrieLoader.Language.PT_PT, SnowballLanguageCase.PORTUGUESE),
SNOWBALL_RUSSIAN_DIRECT(StemmerPatchTrieLoader.Language.RU_RU, SnowballLanguageCase.RUSSIAN),

101
src/test/java/org/egothor/stemmer/benchmark/SnowballLanguageStemmerComparisonBenchmarkTest.java Normal file
View File

@@ -0,0 +1,101 @@
/*******************************************************************************
* Copyright (C) 2026, Leo Galambos
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package org.egothor.stemmer.benchmark;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.openjdk.jmh.annotations.Param;
/**
* Verifies that direct-only Snowball 3.1.0 algorithms cannot enter the Lucene
* SnowballFilter benchmark domain.
*/
final class SnowballLanguageStemmerComparisonBenchmarkTest {
/**
* Verifies the direct and Lucene parameter domains independently.
*
* @throws ReflectiveOperationException if the benchmark state contract changes
*/
@Test
void directAndLuceneParameterDomainsRemainExplicit() throws ReflectiveOperationException {
final Set<String> directCases = parameterValues(
SnowballLanguageStemmerComparisonBenchmark.DirectSharedState.class);
final Set<String> luceneCases = parameterValues(
SnowballLanguageStemmerComparisonBenchmark.SharedState.class);
final Set<String> registeredCases = Arrays.stream(SnowballLanguageCase.values())
.map(Enum::name)
.collect(Collectors.toUnmodifiableSet());
assertEquals(registeredCases, directCases);
assertEquals(17, directCases.size());
assertEquals(14, luceneCases.size());
assertEquals(Set.of("CZECH", "PERSIAN", "POLISH"), difference(directCases, luceneCases));
for (String luceneCase : luceneCases) {
SnowballLanguageCase.valueOf(luceneCase).luceneSnowballName();
}
for (String directOnlyCase : difference(directCases, luceneCases)) {
assertThrows(IllegalStateException.class,
() -> SnowballLanguageCase.valueOf(directOnlyCase).luceneSnowballName());
}
}
/**
* Reads the declared JMH parameter values from a benchmark state.
*
* @param stateClass benchmark state class
* @return immutable parameter-value set
* @throws NoSuchFieldException if the state no longer declares the parameter
*/
private static Set<String> parameterValues(final Class<?> stateClass) throws NoSuchFieldException {
final Field field = stateClass.getField("languageCaseName");
return Set.of(field.getAnnotation(Param.class).value());
}
/**
* Returns the values present in {@code left} but absent from {@code right}.
*
* @param left source set
* @param right excluded set
* @return immutable set difference
*/
private static Set<String> difference(final Set<String> left, final Set<String> right) {
return left.stream().filter(value -> !right.contains(value)).collect(Collectors.toUnmodifiableSet());
}
}

9
src/test/java/org/egothor/stemmer/benchmark/quality/QualityStemmerMatrixTest.java
View File

@@ -59,10 +59,13 @@ final class QualityStemmerMatrixTest {
@Test @DisplayName("Candidate discovery is derived from every JMH quality candidate")
void discoversEveryCandidate() {
final List<Candidate> candidates = QualityStemmerMatrix.candidates();
assertEquals(98, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed.");
assertEquals(102, candidates.size(), "The current adapter-language matrix size changed; report coverage must be reviewed.");
assertTrue(candidates.stream().anyMatch(candidate -> !candidate.name().endsWith("_RADIXOR")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("DA_DK_RADIXOR")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("YI_RADIXOR")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_CZECH_DIRECT")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_PERSIAN_DIRECT")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("SNOWBALL_POLISH_DIRECT")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_POLIMORF_RADIXOR")
&& candidate.resultLanguage().equals("pl-pl-polimorf")));
assertTrue(candidates.stream().anyMatch(candidate -> candidate.name().equals("POLISH_LUCENE_STEMPEL_DIRECT")
@@ -74,7 +77,7 @@ final class QualityStemmerMatrixTest {
void completePublicationSelectionUsesOnlyDefaultModels() {
final List<Candidate> candidates = StemmingQualityApplication.selectCandidates(
EnumSet.allOf(Language.class), "");
assertEquals(92, candidates.size());
assertEquals(95, candidates.size());
assertTrue(candidates.stream().allMatch(candidate ->
candidate.dictionaryModelId().equals(candidate.language().defaultModelId())));
assertTrue(candidates.stream().noneMatch(candidate ->
@@ -95,7 +98,7 @@ final class QualityStemmerMatrixTest {
final Path report = this.temporaryDirectory.resolve("matrix.csv");
QualityReportWriter.writeCsv(report, rows);
final String text = Files.readString(report, StandardCharsets.UTF_8);
assertEquals(197, text.lines().count());
assertEquals(205, text.lines().count());
for (Candidate candidate : QualityStemmerMatrix.candidates()) {
final String prefix = "\"" + candidate.name() + "\",\"" + candidate.resultLanguage()
+ "\",\"\",\"\",\"\",";