feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
This commit is contained in:
179
python/tests/test_standard_models.py
Normal file
179
python/tests/test_standard_models.py
Normal file
@@ -0,0 +1,179 @@
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import nullcontext
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import radixor
|
||||
|
||||
EXPECTED_MODEL_IDS = {
|
||||
"cs-cz-default",
|
||||
"da-dk-default",
|
||||
"de-de-default",
|
||||
"es-es-default",
|
||||
"fa-ir-default",
|
||||
"fi-fi-default",
|
||||
"fr-fr-default",
|
||||
"he-il-default",
|
||||
"hu-hu-default",
|
||||
"it-it-default",
|
||||
"nb-no-default",
|
||||
"nl-nl-default",
|
||||
"nn-no-default",
|
||||
"pl-pl-unimorph",
|
||||
"pt-pt-default",
|
||||
"ru-ru-default",
|
||||
"sv-se-default",
|
||||
"uk-ua-default",
|
||||
"us-uk-default",
|
||||
"yi-default",
|
||||
}
|
||||
|
||||
|
||||
def _manifest() -> dict:
|
||||
ref = resources.files("radixor_models_standard").joinpath("manifest.json")
|
||||
return json.loads(ref.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_standard_manifest_model_set_versions_and_license():
|
||||
manifest = _manifest()
|
||||
assert manifest["catalog_version"] == "2026.1"
|
||||
assert manifest["distribution_version"] == "0.0.0"
|
||||
assert manifest["format"] == {"compression": "gzip", "magic": "EGTR", "version": 7}
|
||||
assert {model["id"] for model in manifest["models"]} == EXPECTED_MODEL_IDS
|
||||
assert "pl-pl-polimorf" not in {model["id"] for model in manifest["models"]}
|
||||
assert {model["version"] for model in manifest["models"]} == {"1.0.0"}
|
||||
assert {model["provenance"]["license"] for model in manifest["models"]} == {
|
||||
"CC-BY-SA-3.0"
|
||||
}
|
||||
|
||||
|
||||
def test_standard_artifact_checksums_and_v7_headers():
|
||||
root = resources.files("radixor_models_standard")
|
||||
for model in _manifest()["models"]:
|
||||
data = root.joinpath("models").joinpath(f"{model['id']}.rxc").read_bytes()
|
||||
assert hashlib.sha256(data).hexdigest() == model["sha256"]
|
||||
assert gzip.decompress(data)[:8] == b"EGTR\x00\x00\x00\x07"
|
||||
|
||||
|
||||
def test_missing_standard_data_package_is_actionable(monkeypatch):
|
||||
original_files = radixor.importlib.resources.files
|
||||
|
||||
def missing(package: str):
|
||||
if package == radixor._STANDARD_PACKAGE:
|
||||
raise ModuleNotFoundError(package)
|
||||
return original_files(package)
|
||||
|
||||
monkeypatch.setattr(radixor.importlib.resources, "files", missing)
|
||||
with pytest.raises(
|
||||
ModuleNotFoundError, match="pip install radixor-models-standard"
|
||||
):
|
||||
radixor.Stemmer("en")
|
||||
|
||||
|
||||
def test_missing_standard_model_is_actionable():
|
||||
with pytest.raises(FileNotFoundError, match="not in the standard Radixor catalog"):
|
||||
radixor.Stemmer("zz-zz-default")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "message"),
|
||||
[
|
||||
(
|
||||
lambda manifest: manifest.update(catalog_version="2027.1"),
|
||||
"incompatible or corrupt",
|
||||
),
|
||||
(
|
||||
lambda manifest: manifest.update(format={"magic": "bad"}),
|
||||
"incompatible or corrupt",
|
||||
),
|
||||
(
|
||||
lambda manifest: manifest.update(distribution_version="2.0.0"),
|
||||
"incompatible or corrupt",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_incompatible_manifest_is_actionable(
|
||||
tmp_path: Path, monkeypatch, mutation, message
|
||||
):
|
||||
manifest = _manifest()
|
||||
mutation(manifest)
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
radixor.Stemmer("en")
|
||||
|
||||
|
||||
def test_checksum_failure_is_detected(tmp_path: Path, monkeypatch):
|
||||
manifest = _manifest()
|
||||
(tmp_path / "models").mkdir()
|
||||
(tmp_path / "models" / "us-uk-default.rxc").write_bytes(b"not the model")
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
|
||||
monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
|
||||
with pytest.raises(RuntimeError, match="SHA-256 validation"):
|
||||
radixor.Stemmer("en")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream_header", "message"),
|
||||
[
|
||||
(b"NOPE\x00\x00\x00\x07", "EGTR format marker"),
|
||||
(b"EGTR\x00\x00\x00\x08", "unsupported compiled format v8"),
|
||||
],
|
||||
)
|
||||
def test_format_marker_and_version_are_validated(
|
||||
tmp_path: Path, monkeypatch, stream_header: bytes, message: str
|
||||
):
|
||||
manifest = _manifest()
|
||||
model = next(item for item in manifest["models"] if item["id"] == "us-uk-default")
|
||||
data = gzip.compress(stream_header, mtime=0)
|
||||
model["sha256"] = hashlib.sha256(data).hexdigest()
|
||||
(tmp_path / "models").mkdir()
|
||||
(tmp_path / "models" / "us-uk-default.rxc").write_bytes(data)
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
|
||||
monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
radixor.Stemmer("en")
|
||||
|
||||
|
||||
def test_invalid_model_id_is_rejected_before_resource_lookup():
|
||||
with pytest.raises(ValueError, match="Invalid Radixor model ID"):
|
||||
radixor.Stemmer("../us-uk-default")
|
||||
Reference in New Issue
Block a user