Radixor/python/tests/test_standard_models.py

###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################

from __future__ import annotations

import gzip
import hashlib
import json
from contextlib import nullcontext
from importlib import resources
from pathlib import Path

import pytest

import radixor

EXPECTED_MODEL_IDS = {
    "cs-cz-default",
    "da-dk-default",
    "de-de-default",
    "es-es-default",
    "fa-ir-default",
    "fi-fi-default",
    "fr-fr-default",
    "he-il-default",
    "hu-hu-default",
    "it-it-default",
    "nb-no-default",
    "nl-nl-default",
    "nn-no-default",
    "pl-pl-unimorph",
    "pt-pt-default",
    "ru-ru-default",
    "sv-se-default",
    "uk-ua-default",
    "us-uk-default",
    "yi-default",
}


def _manifest() -> dict:
    ref = resources.files("radixor_models_standard").joinpath("manifest.json")
    return json.loads(ref.read_text(encoding="utf-8"))


def test_standard_manifest_model_set_versions_and_license():
    manifest = _manifest()
    assert manifest["catalog_version"] == "2026.1"
    assert manifest["distribution_version"] == "0.0.0"
    assert manifest["format"] == {"compression": "gzip", "magic": "EGTR", "version": 7}
    assert {model["id"] for model in manifest["models"]} == EXPECTED_MODEL_IDS
    assert "pl-pl-polimorf" not in {model["id"] for model in manifest["models"]}
    assert {model["version"] for model in manifest["models"]} == {"1.0.0"}
    assert {model["provenance"]["license"] for model in manifest["models"]} == {
        "CC-BY-SA-3.0"
    }


def test_standard_artifact_checksums_and_v7_headers():
    root = resources.files("radixor_models_standard")
    for model in _manifest()["models"]:
        data = root.joinpath("models").joinpath(f"{model['id']}.rxc").read_bytes()
        assert hashlib.sha256(data).hexdigest() == model["sha256"]
        assert gzip.decompress(data)[:8] == b"EGTR\x00\x00\x00\x07"


def test_missing_standard_data_package_is_actionable(monkeypatch):
    original_files = radixor.importlib.resources.files

    def missing(package: str):
        if package == radixor._STANDARD_PACKAGE:
            raise ModuleNotFoundError(package)
        return original_files(package)

    monkeypatch.setattr(radixor.importlib.resources, "files", missing)
    with pytest.raises(
        ModuleNotFoundError, match="pip install radixor-models-standard"
    ):
        radixor.Stemmer("en")


def test_missing_standard_model_is_actionable():
    with pytest.raises(FileNotFoundError, match="not in the standard Radixor catalog"):
        radixor.Stemmer("zz-zz-default")


@pytest.mark.parametrize(
    ("mutation", "message"),
    [
        (
            lambda manifest: manifest.update(catalog_version="2027.1"),
            "incompatible or corrupt",
        ),
        (
            lambda manifest: manifest.update(format={"magic": "bad"}),
            "incompatible or corrupt",
        ),
        (
            lambda manifest: manifest.update(distribution_version="2.0.0"),
            "incompatible or corrupt",
        ),
    ],
)
def test_incompatible_manifest_is_actionable(
    tmp_path: Path, monkeypatch, mutation, message
):
    manifest = _manifest()
    mutation(manifest)
    (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
    monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
    with pytest.raises(RuntimeError, match=message):
        radixor.Stemmer("en")


def test_checksum_failure_is_detected(tmp_path: Path, monkeypatch):
    manifest = _manifest()
    (tmp_path / "models").mkdir()
    (tmp_path / "models" / "us-uk-default.rxc").write_bytes(b"not the model")
    (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
    monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
    monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
    with pytest.raises(RuntimeError, match="SHA-256 validation"):
        radixor.Stemmer("en")


@pytest.mark.parametrize(
    ("stream_header", "message"),
    [
        (b"NOPE\x00\x00\x00\x07", "EGTR format marker"),
        (b"EGTR\x00\x00\x00\x08", "unsupported compiled format v8"),
    ],
)
def test_format_marker_and_version_are_validated(
    tmp_path: Path, monkeypatch, stream_header: bytes, message: str
):
    manifest = _manifest()
    model = next(item for item in manifest["models"] if item["id"] == "us-uk-default")
    data = gzip.compress(stream_header, mtime=0)
    model["sha256"] = hashlib.sha256(data).hexdigest()
    (tmp_path / "models").mkdir()
    (tmp_path / "models" / "us-uk-default.rxc").write_bytes(data)
    (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
    monkeypatch.setattr(radixor.importlib.resources, "files", lambda package: tmp_path)
    monkeypatch.setattr(radixor.importlib.resources, "as_file", nullcontext)
    with pytest.raises(RuntimeError, match=message):
        radixor.Stemmer("en")


def test_invalid_model_id_is_rejected_before_resource_lookup():
    with pytest.raises(ValueError, match="Invalid Radixor model ID"):
        radixor.Stemmer("../us-uk-default")