feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
This commit is contained in:
285
python/scripts/build_standard_models.py
Normal file
285
python/scripts/build_standard_models.py
Normal file
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Regenerate the pure-Python standard model package deterministically.
|
||||
|
||||
The canonical build topology selects ``default`` models. Source dictionaries
|
||||
are compiler inputs only and are never copied into either Python distribution.
|
||||
Run this script with a built Radixor extension importable by the selected
|
||||
Python interpreter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
PYTHON_ROOT = REPOSITORY / "python"
|
||||
BUILD_ROOT = REPOSITORY / "build"
|
||||
SOURCE_PROJECT = PYTHON_ROOT / "models-standard"
|
||||
TOPOLOGY = REPOSITORY / "models" / "model-projects.properties"
|
||||
CATALOG_VERSION = REPOSITORY / "models" / "catalog-version.txt"
|
||||
EXPECTED_FORMAT = {"compression": "gzip", "magic": "EGTR", "version": 7}
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
|
||||
METADATA_FIELDS = (
|
||||
"sourceName",
|
||||
"sourceVersion",
|
||||
"sourceRevision",
|
||||
"sourceProject",
|
||||
"sourceRepository",
|
||||
"sourceDataset",
|
||||
"sourceRevisionStatus",
|
||||
"sourceLicense",
|
||||
"sourceLicenseUri",
|
||||
"sourceAttribution",
|
||||
"sourceVerificationDate",
|
||||
"transformationsSummary",
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _default_models() -> list[str]:
|
||||
entries: dict[str, str] = {}
|
||||
for raw_line in TOPOLOGY.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
model_id, separator, membership = line.partition("=")
|
||||
if not separator or not model_id or membership not in {"default", "optional"}:
|
||||
raise ValueError(f"Invalid model topology line: {raw_line!r}")
|
||||
entries[model_id] = membership
|
||||
model_ids = sorted(
|
||||
model_id for model_id, membership in entries.items() if membership == "default"
|
||||
)
|
||||
if len(model_ids) != 20 or "pl-pl-polimorf" in model_ids:
|
||||
raise ValueError(
|
||||
"Standard Python catalog must contain 20 defaults and exclude pl-pl-polimorf"
|
||||
)
|
||||
return model_ids
|
||||
|
||||
|
||||
def _gradle_metadata(path: Path) -> dict[str, str]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
result: dict[str, str] = {}
|
||||
for field in METADATA_FIELDS:
|
||||
match = re.search(rf"^\s*{field}\s*=\s*'([^']*)'\s*$", text, re.MULTILINE)
|
||||
if match is None:
|
||||
raise ValueError(f"Missing {field} in {path}")
|
||||
result[field] = match.group(1)
|
||||
return result
|
||||
|
||||
|
||||
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
|
||||
if count != 1:
|
||||
raise ValueError(f"Expected exactly one version field in {path}")
|
||||
path.write_text(updated, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def _stage_project(output: Path, distribution_version: str) -> Path:
|
||||
project = output.resolve()
|
||||
try:
|
||||
project.relative_to(BUILD_ROOT.resolve())
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Generated model project must be below {BUILD_ROOT}") from exc
|
||||
if project == BUILD_ROOT.resolve():
|
||||
raise ValueError("Generated model project cannot be the build root")
|
||||
if project.exists():
|
||||
shutil.rmtree(project)
|
||||
project.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(
|
||||
SOURCE_PROJECT,
|
||||
project,
|
||||
ignore=shutil.ignore_patterns(
|
||||
"build",
|
||||
"dist",
|
||||
"*.egg-info",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"manifest.json",
|
||||
"*.rxc",
|
||||
"NOTICE-model-data.txt",
|
||||
),
|
||||
)
|
||||
_replace_once(
|
||||
project / "pyproject.toml",
|
||||
r'^version = "0\.0\.0"$',
|
||||
f'version = "{distribution_version}"',
|
||||
)
|
||||
_replace_once(
|
||||
project / "radixor_models_standard" / "__init__.py",
|
||||
r'^__version__ = "0\.0\.0"$',
|
||||
f'__version__ = "{distribution_version}"',
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def _model_manifest(
|
||||
model_id: str,
|
||||
compile_model: Any,
|
||||
package_root: Path,
|
||||
reproducibility_directory: Path,
|
||||
) -> dict[str, Any]:
|
||||
project = REPOSITORY / "models" / model_id
|
||||
source = project / "src" / "modelInput" / "stemmer.gz"
|
||||
notice = project / "src" / "modelInput" / "NOTICE-model-data.txt"
|
||||
version = (project / "model-version.txt").read_text(encoding="utf-8").strip()
|
||||
if not source.is_file() or not notice.is_file() or not version:
|
||||
raise FileNotFoundError(f"Incomplete canonical model project: {project}")
|
||||
|
||||
destination = package_root / "models" / f"{model_id}.rxc"
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
compile_model(str(source), str(destination), language=model_id)
|
||||
reproduction = reproducibility_directory / destination.name
|
||||
compile_model(str(source), str(reproduction), language=model_id)
|
||||
if destination.read_bytes() != reproduction.read_bytes():
|
||||
raise ValueError(f"Non-deterministic compiled model output for {model_id}")
|
||||
|
||||
notice_destination = package_root / "notices" / model_id / notice.name
|
||||
notice_destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(notice, notice_destination)
|
||||
|
||||
metadata = _gradle_metadata(project / "build.gradle")
|
||||
if metadata["sourceLicense"] != "CC-BY-SA-3.0":
|
||||
raise ValueError(f"Standard model {model_id} must be CC-BY-SA-3.0")
|
||||
return {
|
||||
"file": f"models/{model_id}.rxc",
|
||||
"id": model_id,
|
||||
"notice": f"notices/{model_id}/{notice.name}",
|
||||
"provenance": {
|
||||
"attribution": metadata["sourceAttribution"],
|
||||
"dataset": metadata["sourceDataset"],
|
||||
"license": metadata["sourceLicense"],
|
||||
"license_uri": metadata["sourceLicenseUri"],
|
||||
"repository": metadata["sourceRepository"],
|
||||
"revision": metadata["sourceRevision"],
|
||||
"revision_status": metadata["sourceRevisionStatus"],
|
||||
"source_name": metadata["sourceName"],
|
||||
"source_project": metadata["sourceProject"],
|
||||
"source_version": metadata["sourceVersion"],
|
||||
"transformations": metadata["transformationsSummary"],
|
||||
"verification_date": metadata["sourceVerificationDate"],
|
||||
},
|
||||
"sha256": _sha256(destination),
|
||||
"source": {
|
||||
"path": f"models/{model_id}/src/modelInput/stemmer.gz",
|
||||
"sha256": _sha256(source),
|
||||
},
|
||||
"version": version,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", required=True, type=Path)
|
||||
parser.add_argument("--distribution-version", required=True)
|
||||
args = parser.parse_args()
|
||||
if VERSION.fullmatch(args.distribution_version) is None:
|
||||
raise SystemExit(
|
||||
f"Invalid standard-model distribution version: {args.distribution_version!r}"
|
||||
)
|
||||
|
||||
try:
|
||||
from radixor import compile as compile_model
|
||||
except ImportError as exc:
|
||||
print(
|
||||
"error: build the Radixor extension first (for example, "
|
||||
"maturin develop --release) and rerun this script",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(f"detail: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
project = _stage_project(args.project, args.distribution_version)
|
||||
package_root = project / "radixor_models_standard"
|
||||
model_ids = _default_models()
|
||||
expected_files = {f"{model_id}.rxc" for model_id in model_ids}
|
||||
models_directory = package_root / "models"
|
||||
models_directory.mkdir(parents=True, exist_ok=True)
|
||||
for stale in models_directory.glob("*.rxc"):
|
||||
if stale.name not in expected_files:
|
||||
stale.unlink()
|
||||
notices_directory = package_root / "notices"
|
||||
notices_directory.mkdir(parents=True, exist_ok=True)
|
||||
for stale in notices_directory.iterdir():
|
||||
if stale.is_dir() and stale.name not in model_ids:
|
||||
shutil.rmtree(stale)
|
||||
|
||||
reproducibility_root = BUILD_ROOT / "python" / "tmp"
|
||||
reproducibility_root.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="radixor-model-reproducibility-", dir=reproducibility_root
|
||||
) as temporary:
|
||||
models = [
|
||||
_model_manifest(
|
||||
model_id,
|
||||
compile_model,
|
||||
package_root,
|
||||
Path(temporary),
|
||||
)
|
||||
for model_id in model_ids
|
||||
]
|
||||
manifest = {
|
||||
"catalog_version": CATALOG_VERSION.read_text(encoding="utf-8").strip(),
|
||||
"distribution_version": args.distribution_version,
|
||||
"format": EXPECTED_FORMAT,
|
||||
"models": models,
|
||||
"schema_version": 1,
|
||||
"topology": "models/model-projects.properties",
|
||||
}
|
||||
manifest_path = package_root / "manifest.json"
|
||||
with manifest_path.open("w", encoding="utf-8", newline="\n") as stream:
|
||||
stream.write(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
print(f"generated {len(models)} standard compiled models in {package_root}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user