feat(python): add native distribution and release infrastructure

- add the Rust-backed Python API with PyStemmer compatibility
- distribute standard compiled models as a separate Python package
- generate model artifacts during builds instead of storing them in Git
- add GitHub release and Pages-backed package index workflows
- add Python tests, benchmarks, documentation, and Gradle integration
- refresh the documentation site, branding, and language benchmarks
This commit is contained in:
2026-08-10 22:34:32 +02:00
parent b45e143c84
commit 5e3d3c7c7d
139 changed files with 11420 additions and 747 deletions

View File

@@ -0,0 +1,171 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Validate and assemble the exact files allowed in a Python GitHub Release."""
from __future__ import annotations
import argparse
import hashlib
import re
import shutil
from pathlib import Path
from verify_distributions import (
_verify_main_sdist,
_verify_main_wheel,
_verify_standard_sdist,
_verify_standard_wheel,
)
REPOSITORY = Path(__file__).resolve().parents[2]
BUILD_ROOT = (REPOSITORY / "build").resolve()
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
def _release_files(root: Path) -> list[Path]:
files = sorted(
path
for path in root.rglob("*")
if path.is_file()
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
)
names = [path.name for path in files]
if len(names) != len(set(names)):
raise ValueError(f"Duplicate release filenames: {names}")
return files
def _require_one(files: list[Path], predicate, description: str) -> Path:
matches = [path for path in files if predicate(path.name)]
if len(matches) != 1:
raise ValueError(
f"Expected one {description}, found {[path.name for path in matches]}"
)
return matches[0]
def _validate_native(files: list[Path], version: str) -> None:
prefix = f"radixor-{version}-cp39-abi3-"
wheels = [path for path in files if path.suffix == ".whl"]
sdist = _require_one(
files, lambda name: name == f"radixor-{version}.tar.gz", "native sdist"
)
if len(wheels) != 4:
raise ValueError(
f"Expected four native wheels, found {[path.name for path in wheels]}"
)
expected = {
"linux-x86_64": lambda name: name.startswith(prefix)
and "manylinux" in name
and name.endswith("x86_64.whl"),
"linux-aarch64": lambda name: name.startswith(prefix)
and "manylinux" in name
and name.endswith("aarch64.whl"),
"macos-universal2": lambda name: name.startswith(prefix)
and "macosx" in name
and name.endswith("universal2.whl"),
"windows-x86_64": lambda name: name == f"{prefix}win_amd64.whl",
}
for description, predicate in expected.items():
_require_one(wheels, predicate, description)
for wheel in wheels:
_verify_main_wheel(wheel, version)
_verify_main_sdist(sdist, version)
def _validate_models(files: list[Path], version: str) -> None:
wheel = _require_one(
files,
lambda name: name == f"radixor_models_standard-{version}-py3-none-any.whl",
"standard-model wheel",
)
sdist = _require_one(
files,
lambda name: name == f"radixor_models_standard-{version}.tar.gz",
"standard-model sdist",
)
if len(files) != 2:
raise ValueError(
f"Unexpected standard-model release files: {[path.name for path in files]}"
)
_verify_standard_wheel(wheel, version)
_verify_standard_sdist(sdist, version)
def _output_directory(path: Path) -> Path:
output = path.resolve()
try:
output.relative_to(BUILD_ROOT)
except ValueError as exc:
raise ValueError(f"Release output must be below {BUILD_ROOT}") from exc
if output == BUILD_ROOT:
raise ValueError("Release output cannot be the build root")
if output.exists():
shutil.rmtree(output)
output.mkdir(parents=True)
return output
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("distribution", choices=("native", "models-standard"))
parser.add_argument("version")
parser.add_argument("artifacts", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
if VERSION.fullmatch(args.version) is None:
raise SystemExit(f"Invalid stable release version: {args.version!r}")
files = _release_files(args.artifacts.resolve())
if args.distribution == "native":
_validate_native(files, args.version)
else:
_validate_models(files, args.version)
output = _output_directory(args.output)
for source in files:
shutil.copy2(source, output / source.name)
checksums = [
f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}"
for path in sorted(output.iterdir())
if path.is_file()
]
(output / "SHA256SUMS").write_text(
"\n".join(checksums) + "\n", encoding="utf-8", newline="\n"
)
print(f"assembled {len(files)} release artifacts in {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Build the standard-model wheel and sdist through its declared backend."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import tempfile
from pathlib import Path
EXPECTED_MODEL_COUNT = 20
def _ignore_build_artifacts(_directory: str, names: list[str]) -> set[str]:
"""Exclude local build state from the isolated distribution source tree."""
ignored = {
name
for name in names
if name in {"build", "dist", "__pycache__"}
or name.endswith((".egg-info", ".pyc", ".pyo"))
}
return ignored
def _validate_generated_project(project: Path) -> None:
"""Reject an ungenerated or incomplete standard-model project."""
package = project / "radixor_models_standard"
manifest_path = package / "manifest.json"
if not manifest_path.is_file():
raise ValueError(
"standard-model manifest is missing; run build_standard_models.py first"
)
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
model_ids = {
model["id"] for model in manifest["models"] if isinstance(model, dict)
}
except (json.JSONDecodeError, KeyError, TypeError) as exc:
raise ValueError("standard-model manifest is invalid") from exc
compiled = {path.stem for path in (package / "models").glob("*.rxc")}
notices = {
path.parent.name
for path in (package / "notices").glob("*/NOTICE-model-data.txt")
}
if len(model_ids) != EXPECTED_MODEL_COUNT:
raise ValueError(
f"expected {EXPECTED_MODEL_COUNT} standard models, found {len(model_ids)}"
)
if compiled != model_ids:
raise ValueError("compiled standard models do not match the manifest")
if notices != model_ids:
raise ValueError("standard-model notices do not match the manifest")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True, type=Path)
parser.add_argument("--outdir", required=True, type=Path)
args = parser.parse_args()
try:
from setuptools.build_meta import build_sdist, build_wheel
except ImportError as exc:
raise SystemExit(
"setuptools>=77 is required to build radixor-models-standard"
) from exc
source_project = args.project.resolve()
try:
_validate_generated_project(source_project)
except ValueError as exc:
parser.error(str(exc))
output = args.outdir.resolve()
output.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="radixor-models-build-") as temporary:
project = Path(temporary, "project")
shutil.copytree(
source_project,
project,
ignore=_ignore_build_artifacts,
)
os.chdir(project)
wheel = build_wheel(str(output))
sdist = build_sdist(str(output))
print(f"built {wheel} and {sdist}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,285 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Regenerate the pure-Python standard model package deterministically.
The canonical build topology selects ``default`` models. Source dictionaries
are compiler inputs only and are never copied into either Python distribution.
Run this script with a built Radixor extension importable by the selected
Python interpreter.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Any
REPOSITORY = Path(__file__).resolve().parents[2]
PYTHON_ROOT = REPOSITORY / "python"
BUILD_ROOT = REPOSITORY / "build"
SOURCE_PROJECT = PYTHON_ROOT / "models-standard"
TOPOLOGY = REPOSITORY / "models" / "model-projects.properties"
CATALOG_VERSION = REPOSITORY / "models" / "catalog-version.txt"
EXPECTED_FORMAT = {"compression": "gzip", "magic": "EGTR", "version": 7}
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
METADATA_FIELDS = (
"sourceName",
"sourceVersion",
"sourceRevision",
"sourceProject",
"sourceRepository",
"sourceDataset",
"sourceRevisionStatus",
"sourceLicense",
"sourceLicenseUri",
"sourceAttribution",
"sourceVerificationDate",
"transformationsSummary",
)
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _default_models() -> list[str]:
entries: dict[str, str] = {}
for raw_line in TOPOLOGY.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
model_id, separator, membership = line.partition("=")
if not separator or not model_id or membership not in {"default", "optional"}:
raise ValueError(f"Invalid model topology line: {raw_line!r}")
entries[model_id] = membership
model_ids = sorted(
model_id for model_id, membership in entries.items() if membership == "default"
)
if len(model_ids) != 20 or "pl-pl-polimorf" in model_ids:
raise ValueError(
"Standard Python catalog must contain 20 defaults and exclude pl-pl-polimorf"
)
return model_ids
def _gradle_metadata(path: Path) -> dict[str, str]:
text = path.read_text(encoding="utf-8")
result: dict[str, str] = {}
for field in METADATA_FIELDS:
match = re.search(rf"^\s*{field}\s*=\s*'([^']*)'\s*$", text, re.MULTILINE)
if match is None:
raise ValueError(f"Missing {field} in {path}")
result[field] = match.group(1)
return result
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
text = path.read_text(encoding="utf-8")
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
if count != 1:
raise ValueError(f"Expected exactly one version field in {path}")
path.write_text(updated, encoding="utf-8", newline="\n")
def _stage_project(output: Path, distribution_version: str) -> Path:
project = output.resolve()
try:
project.relative_to(BUILD_ROOT.resolve())
except ValueError as exc:
raise ValueError(f"Generated model project must be below {BUILD_ROOT}") from exc
if project == BUILD_ROOT.resolve():
raise ValueError("Generated model project cannot be the build root")
if project.exists():
shutil.rmtree(project)
project.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(
SOURCE_PROJECT,
project,
ignore=shutil.ignore_patterns(
"build",
"dist",
"*.egg-info",
"__pycache__",
"*.pyc",
"manifest.json",
"*.rxc",
"NOTICE-model-data.txt",
),
)
_replace_once(
project / "pyproject.toml",
r'^version = "0\.0\.0"$',
f'version = "{distribution_version}"',
)
_replace_once(
project / "radixor_models_standard" / "__init__.py",
r'^__version__ = "0\.0\.0"$',
f'__version__ = "{distribution_version}"',
)
return project
def _model_manifest(
model_id: str,
compile_model: Any,
package_root: Path,
reproducibility_directory: Path,
) -> dict[str, Any]:
project = REPOSITORY / "models" / model_id
source = project / "src" / "modelInput" / "stemmer.gz"
notice = project / "src" / "modelInput" / "NOTICE-model-data.txt"
version = (project / "model-version.txt").read_text(encoding="utf-8").strip()
if not source.is_file() or not notice.is_file() or not version:
raise FileNotFoundError(f"Incomplete canonical model project: {project}")
destination = package_root / "models" / f"{model_id}.rxc"
destination.parent.mkdir(parents=True, exist_ok=True)
compile_model(str(source), str(destination), language=model_id)
reproduction = reproducibility_directory / destination.name
compile_model(str(source), str(reproduction), language=model_id)
if destination.read_bytes() != reproduction.read_bytes():
raise ValueError(f"Non-deterministic compiled model output for {model_id}")
notice_destination = package_root / "notices" / model_id / notice.name
notice_destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(notice, notice_destination)
metadata = _gradle_metadata(project / "build.gradle")
if metadata["sourceLicense"] != "CC-BY-SA-3.0":
raise ValueError(f"Standard model {model_id} must be CC-BY-SA-3.0")
return {
"file": f"models/{model_id}.rxc",
"id": model_id,
"notice": f"notices/{model_id}/{notice.name}",
"provenance": {
"attribution": metadata["sourceAttribution"],
"dataset": metadata["sourceDataset"],
"license": metadata["sourceLicense"],
"license_uri": metadata["sourceLicenseUri"],
"repository": metadata["sourceRepository"],
"revision": metadata["sourceRevision"],
"revision_status": metadata["sourceRevisionStatus"],
"source_name": metadata["sourceName"],
"source_project": metadata["sourceProject"],
"source_version": metadata["sourceVersion"],
"transformations": metadata["transformationsSummary"],
"verification_date": metadata["sourceVerificationDate"],
},
"sha256": _sha256(destination),
"source": {
"path": f"models/{model_id}/src/modelInput/stemmer.gz",
"sha256": _sha256(source),
},
"version": version,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True, type=Path)
parser.add_argument("--distribution-version", required=True)
args = parser.parse_args()
if VERSION.fullmatch(args.distribution_version) is None:
raise SystemExit(
f"Invalid standard-model distribution version: {args.distribution_version!r}"
)
try:
from radixor import compile as compile_model
except ImportError as exc:
print(
"error: build the Radixor extension first (for example, "
"maturin develop --release) and rerun this script",
file=sys.stderr,
)
print(f"detail: {exc}", file=sys.stderr)
return 2
project = _stage_project(args.project, args.distribution_version)
package_root = project / "radixor_models_standard"
model_ids = _default_models()
expected_files = {f"{model_id}.rxc" for model_id in model_ids}
models_directory = package_root / "models"
models_directory.mkdir(parents=True, exist_ok=True)
for stale in models_directory.glob("*.rxc"):
if stale.name not in expected_files:
stale.unlink()
notices_directory = package_root / "notices"
notices_directory.mkdir(parents=True, exist_ok=True)
for stale in notices_directory.iterdir():
if stale.is_dir() and stale.name not in model_ids:
shutil.rmtree(stale)
reproducibility_root = BUILD_ROOT / "python" / "tmp"
reproducibility_root.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix="radixor-model-reproducibility-", dir=reproducibility_root
) as temporary:
models = [
_model_manifest(
model_id,
compile_model,
package_root,
Path(temporary),
)
for model_id in model_ids
]
manifest = {
"catalog_version": CATALOG_VERSION.read_text(encoding="utf-8").strip(),
"distribution_version": args.distribution_version,
"format": EXPECTED_FORMAT,
"models": models,
"schema_version": 1,
"topology": "models/model-projects.properties",
}
manifest_path = package_root / "manifest.json"
with manifest_path.open("w", encoding="utf-8", newline="\n") as stream:
stream.write(
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
)
print(f"generated {len(models)} standard compiled models in {package_root}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,132 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Create an isolated, tag-versioned Python release source tree."""
from __future__ import annotations
import argparse
import re
import shutil
from pathlib import Path
REPOSITORY = Path(__file__).resolve().parents[2]
PYTHON_ROOT = REPOSITORY / "python"
BUILD_ROOT = REPOSITORY / "build"
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
text = path.read_text(encoding="utf-8")
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
if count != 1:
raise ValueError(f"Expected exactly one version field in {path}")
path.write_text(updated, encoding="utf-8", newline="\n")
def _prepare_output(path: Path) -> Path:
output = path.resolve()
try:
output.relative_to(BUILD_ROOT.resolve())
except ValueError as exc:
raise ValueError(f"Release staging output must be below {BUILD_ROOT}") from exc
if output == BUILD_ROOT.resolve():
raise ValueError("Release staging output cannot be the build root")
if output.exists():
shutil.rmtree(output)
output.parent.mkdir(parents=True, exist_ok=True)
return output
def _copy_native(output: Path, version: str) -> None:
shutil.copytree(
PYTHON_ROOT,
output,
ignore=shutil.ignore_patterns(
".gitignore",
".pytest_cache",
".ruff_cache",
".venv",
"__pycache__",
"*.pyc",
"_radixor*.dll",
"_radixor*.dylib",
"_radixor*.pyd",
"_radixor*.so",
"_native*.dll",
"_native*.dylib",
"_native*.pyd",
"_native*.so",
"benchmarks",
"dist",
"models",
"models-standard",
"target",
"tests",
),
)
_replace_once(
output / "pyproject.toml",
r'^version = "0\.0\.0"$',
f'version = "{version}"',
)
_replace_once(
output / "Cargo.toml",
r'^version = "0\.0\.0"$',
f'version = "{version}"',
)
lock = output / "Cargo.lock"
lock_text = lock.read_text(encoding="utf-8")
pattern = r'(\[\[package\]\]\nname = "radixor"\n)version = "0\.0\.0"'
lock_text, count = re.subn(pattern, rf'\g<1>version = "{version}"', lock_text)
if count != 1:
raise ValueError("Expected exactly one radixor package entry in Cargo.lock")
lock.write_text(lock_text, encoding="utf-8", newline="\n")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("distribution", choices=("native",))
parser.add_argument("version")
parser.add_argument("output", type=Path)
args = parser.parse_args()
if VERSION.fullmatch(args.version) is None:
raise SystemExit(f"Invalid stable release version: {args.version!r}")
output = _prepare_output(args.output)
_copy_native(output, args.version)
print(f"prepared {args.distribution} {args.version} in {output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,173 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Update Radixor's static PEP 503 index with verified GitHub Release assets."""
from __future__ import annotations
import argparse
import hashlib
import html
import re
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import quote
PACKAGES = ("radixor", "radixor-models-standard")
REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z")
SHA256 = re.compile(r"[0-9a-f]{64}\Z")
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
class _AnchorParser(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.anchors: dict[str, str] = {}
self._href: str | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag == "a":
self._href = dict(attrs).get("href")
def handle_data(self, data: str) -> None:
if self._href is not None and data.strip():
self.anchors[data.strip()] = self._href
def handle_endtag(self, tag: str) -> None:
if tag == "a":
self._href = None
def _existing_links(path: Path, expected_prefix: str) -> dict[str, str]:
if not path.exists():
return {}
parser = _AnchorParser()
parser.feed(path.read_text(encoding="utf-8"))
for filename, href in parser.anchors.items():
base, separator, digest = href.rpartition("#sha256=")
if (
not separator
or not base.startswith(expected_prefix)
or SHA256.fullmatch(digest) is None
or base.rsplit("/", 1)[-1] != quote(filename, safe="._-")
):
raise ValueError(f"Existing package-index link is unsafe: {href!r}")
return parser.anchors
def _render_root(root: Path) -> None:
links = "\n".join(
f' <a href="{html.escape(package)}/">{html.escape(package)}</a><br>'
for package in PACKAGES
)
(root / "index.html").write_text(
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
'content="1.0"><title>Radixor Python packages</title></head>\n'
f"<body>\n{links}\n</body></html>\n",
encoding="utf-8",
newline="\n",
)
def _render_project(path: Path, package: str, links: dict[str, str]) -> None:
anchors = "\n".join(
f' <a href="{html.escape(href, quote=True)}" '
f'data-requires-python="&gt;=3.9">{html.escape(filename)}</a><br>'
for filename, href in sorted(links.items())
)
path.write_text(
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
f'content="1.0"><title>Links for {html.escape(package)}</title></head>\n'
f"<body>\n{anchors}\n</body></html>\n",
encoding="utf-8",
newline="\n",
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--root", required=True, type=Path)
parser.add_argument("--repository", required=True)
parser.add_argument("--package", required=True, choices=PACKAGES)
parser.add_argument("--version", required=True)
parser.add_argument("--tag", required=True)
parser.add_argument("--artifacts", required=True, type=Path)
args = parser.parse_args()
if REPOSITORY.fullmatch(args.repository) is None:
raise SystemExit("Invalid GitHub repository identity")
if VERSION.fullmatch(args.version) is None:
raise SystemExit("Invalid stable release version")
expected_tag = (
f"python@{args.version}"
if args.package == "radixor"
else f"python-models-standard@{args.version}"
)
if args.tag != expected_tag:
raise SystemExit(f"Tag {args.tag!r} does not match {expected_tag!r}")
root = args.root.resolve()
if root.name != "simple" or root.parent.name != "python":
raise SystemExit("The package index must end in python/simple")
root.mkdir(parents=True, exist_ok=True)
project = root / args.package
project.mkdir(exist_ok=True)
page = project / "index.html"
prefix = f"https://github.com/{args.repository}/releases/download/"
links = _existing_links(page, prefix)
release_prefix = f"{prefix}{quote(args.tag, safe='@._-')}/"
artifacts = sorted(
path
for path in args.artifacts.iterdir()
if path.is_file()
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
)
if not artifacts:
raise SystemExit("No package artifacts were provided")
for artifact in artifacts:
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
href = f"{release_prefix}{quote(artifact.name, safe='._-')}#sha256={digest}"
previous = links.get(artifact.name)
if previous is not None and previous != href:
raise ValueError(
f"Refusing to replace existing index entry {artifact.name}"
)
links[artifact.name] = href
_render_root(root)
_render_project(page, args.package, links)
print(f"indexed {len(artifacts)} artifacts for {args.package} {args.version}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,598 @@
#!/usr/bin/env python3
###############################################################################
# Copyright (C) 2026, Leo Galambos
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###############################################################################
"""Verify Radixor's native and standard-model release archives."""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import os
import stat
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from email.parser import BytesParser
from pathlib import Path, PurePosixPath
EXPECTED_DEFAULT_COUNT = 20
EXPECTED_DEPENDENCY = "radixor-models-standard>=1.0,<2.0"
REPOSITORY = Path(__file__).resolve().parents[2]
EXPECTED_NATIVE_LICENSE = REPOSITORY.joinpath("LICENSE").read_bytes()
EXPECTED_MODEL_IDS = {
"cs-cz-default",
"da-dk-default",
"de-de-default",
"es-es-default",
"fa-ir-default",
"fi-fi-default",
"fr-fr-default",
"he-il-default",
"hu-hu-default",
"it-it-default",
"nb-no-default",
"nl-nl-default",
"nn-no-default",
"pl-pl-unimorph",
"pt-pt-default",
"ru-ru-default",
"sv-se-default",
"uk-ua-default",
"us-uk-default",
"yi-default",
}
STANDARD_NAME = "radixor-models-standard"
STANDARD_VERSION = "0.0.0"
MAIN_NAME = "radixor"
MAIN_VERSION = "0.0.0"
def _one(directory: Path, pattern: str) -> Path:
matches = sorted(directory.glob(pattern))
if len(matches) != 1:
raise ValueError(
f"Expected exactly one {pattern} in {directory}, found {len(matches)}"
)
return matches[0]
def _assert_no_repository_temp_gzip() -> None:
offenders = sorted(
path.name for path in REPOSITORY.glob("tmp*.gz") if path.is_file()
)
if offenders:
raise ValueError(
"Repository root contains unmanaged Python temporary gzip files; "
f"remove them and fix the producing workflow: {offenders}"
)
def _wheel_metadata(archive: zipfile.ZipFile) -> object:
names = [
name for name in archive.namelist() if name.endswith(".dist-info/METADATA")
]
if len(names) != 1:
raise ValueError("Wheel must contain exactly one METADATA file")
return BytesParser().parsebytes(archive.read(names[0]))
def _normalized_member_name(name: str, archive_name: str) -> str:
if not name or "\\" in name or name.startswith("/"):
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
stripped = name[:-1] if name.endswith("/") else name
parts = stripped.split("/")
if not stripped or any(part in {"", ".", ".."} for part in parts):
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
return "/".join(parts)
def _validate_zip_members(archive: zipfile.ZipFile, archive_name: str) -> list[str]:
names: list[str] = []
seen: set[str] = set()
for info in archive.infolist():
name = _normalized_member_name(info.filename, archive_name)
if name in seen:
raise ValueError(f"{archive_name} contains duplicate member: {name}")
seen.add(name)
mode = (info.external_attr >> 16) & 0xFFFF
if stat.S_ISLNK(mode):
raise ValueError(f"{archive_name} contains symlink-like ZIP member: {name}")
names.append(name)
return names
def _validate_tar_members(archive: tarfile.TarFile, archive_name: str) -> list[str]:
names: list[str] = []
seen: set[str] = set()
for member in archive.getmembers():
name = _normalized_member_name(member.name, archive_name)
if name in seen:
raise ValueError(f"{archive_name} contains duplicate member: {name}")
seen.add(name)
if not (member.isfile() or member.isdir()):
raise ValueError(f"{archive_name} contains unsafe special member: {name}")
names.append(name)
return names
def _assert_distribution_metadata(
metadata: object, name: str, version: str, archive_name: str
) -> None:
if metadata.get("Name") != name or metadata.get("Version") != version:
raise ValueError(
f"{archive_name} has unexpected distribution identity: "
f"{metadata.get('Name')} {metadata.get('Version')}"
)
def _tar_file_bytes(archive: tarfile.TarFile, name: str) -> bytes:
member = archive.getmember(name)
stream = archive.extractfile(member)
if stream is None:
raise ValueError(f"Cannot read archive member: {name}")
return stream.read()
def _validate_standard_manifest(
manifest: object, expected_version: str = STANDARD_VERSION
) -> list[dict]:
if not isinstance(manifest, dict) or set(manifest) != {
"catalog_version",
"distribution_version",
"format",
"models",
"schema_version",
"topology",
}:
raise ValueError("Standard model manifest has an invalid top-level schema")
if (
manifest["schema_version"] != 1
or manifest["catalog_version"] != "2026.1"
or manifest["distribution_version"] != expected_version
or manifest["topology"] != "models/model-projects.properties"
or manifest["format"] != {"compression": "gzip", "magic": "EGTR", "version": 7}
or not isinstance(manifest["models"], list)
):
raise ValueError("Standard model manifest has incompatible catalog metadata")
models = manifest["models"]
if {
model.get("id") for model in models if isinstance(model, dict)
} != EXPECTED_MODEL_IDS:
raise ValueError(
"Standard model manifest does not match the default model topology"
)
if len(models) != len(EXPECTED_MODEL_IDS):
raise ValueError("Standard model manifest contains duplicate model entries")
provenance_keys = {
"attribution",
"dataset",
"license",
"license_uri",
"repository",
"revision",
"revision_status",
"source_name",
"source_project",
"source_version",
"transformations",
"verification_date",
}
for model in models:
if set(model) != {
"file",
"id",
"notice",
"provenance",
"sha256",
"source",
"version",
}:
raise ValueError("Standard model manifest contains an invalid model entry")
model_id = model["id"]
if (
model["file"] != f"models/{model_id}.rxc"
or model["notice"] != f"notices/{model_id}/NOTICE-model-data.txt"
or model["version"] != "1.0.0"
or not isinstance(model["sha256"], str)
or len(model["sha256"]) != 64
or set(model["provenance"]) != provenance_keys
or model["provenance"]["license"] != "CC-BY-SA-3.0"
or set(model["source"]) != {"path", "sha256"}
or model["source"]["path"] != f"models/{model_id}/src/modelInput/stemmer.gz"
):
raise ValueError(f"Standard model manifest entry is invalid: {model_id}")
if not all(isinstance(value, str) for value in model["provenance"].values()):
raise ValueError(f"Standard model provenance is invalid: {model_id}")
return models
def _assert_no_source_dictionaries(names: list[str], archive_name: str) -> None:
offenders = [
name
for name in names
if name.endswith("stemmer.gz") or "/radixor/models/" in name
]
if offenders:
raise ValueError(
f"{archive_name} contains forbidden source dictionaries: {offenders}"
)
def _assert_no_standard_model_payload(names: list[str], archive_name: str) -> None:
offenders = []
for name in names:
parts = PurePosixPath(name).parts
if (
name.endswith(".rxc")
or "models-standard" in parts
or any(part.startswith("radixor_models_standard") for part in parts)
):
offenders.append(name)
if offenders:
raise ValueError(
f"{archive_name} contains standard-model payload owned by "
f"radixor-models-standard: {offenders}"
)
def _assert_no_local_build_outputs(names: list[str], archive_name: str) -> None:
offenders = []
for name in names:
path = PurePosixPath(name)
parts = path.parts
is_benchmark_result = (
"benchmarks" in parts
and path.name.startswith("results")
and path.suffix in {".csv", ".json"}
)
if "dist" in parts or is_benchmark_result:
offenders.append(name)
if offenders:
raise ValueError(
f"{archive_name} contains local distribution or benchmark outputs: {offenders}"
)
def _assert_no_python_cache(
names: list[str], archive_name: str, *, allow_native_pyd: bool = False
) -> None:
offenders = []
for name in names:
path = PurePosixPath(name)
is_native_module = (
allow_native_pyd
and path.suffix == ".pyd"
and path.name.startswith("_radixor")
)
if (
"__pycache__" in path.parts
or path.suffix in {".pyc", ".pyo"}
or (path.suffix == ".pyd" and not is_native_module)
):
offenders.append(name)
if offenders:
raise ValueError(f"{archive_name} contains Python cache artifacts: {offenders}")
def _assert_main_dependency(metadata, archive_name: str) -> None:
requirements = metadata.get_all("Requires-Dist", [])
normalized = {requirement.replace(" ", "") for requirement in requirements}
if EXPECTED_DEPENDENCY not in normalized:
raise ValueError(
f"Missing compatible standard-model dependency in {archive_name}: "
f"{requirements}"
)
def _verify_main_wheel(path: Path, expected_version: str = MAIN_VERSION) -> None:
with zipfile.ZipFile(path) as archive:
names = _validate_zip_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_standard_model_payload(names, path.name)
_assert_no_python_cache(names, path.name, allow_native_pyd=True)
metadata = _wheel_metadata(archive)
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
if metadata.get("License-Expression") != "BSD-3-Clause":
raise ValueError(
f"{path.name} must declare License-Expression: BSD-3-Clause"
)
license_names = [
name for name in names if name.endswith(".dist-info/licenses/LICENSE")
]
if (
len(license_names) != 1
or archive.read(license_names[0]) != EXPECTED_NATIVE_LICENSE
):
raise ValueError(
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
)
_assert_main_dependency(metadata, path.name)
def _verify_main_sdist(path: Path, expected_version: str = MAIN_VERSION) -> None:
with tarfile.open(path, "r:gz") as archive:
names = _validate_tar_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_standard_model_payload(names, path.name)
_assert_no_local_build_outputs(names, path.name)
_assert_no_python_cache(names, path.name)
license_members = [
member
for member in archive.getmembers()
if PurePosixPath(member.name).name == "LICENSE" and member.isfile()
]
if len(license_members) != 1:
raise ValueError(
f"{path.name} must contain exactly one BSD-3-Clause LICENSE"
)
license_stream = archive.extractfile(license_members[0])
if license_stream is None or license_stream.read() != EXPECTED_NATIVE_LICENSE:
raise ValueError(
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
)
root = f"radixor-{expected_version}"
metadata = BytesParser().parsebytes(
_tar_file_bytes(archive, f"{root}/PKG-INFO")
)
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
_assert_main_dependency(metadata, path.name)
def _verify_standard_wheel(
path: Path, expected_version: str = STANDARD_VERSION
) -> None:
if not path.name.endswith("-py3-none-any.whl"):
raise ValueError(f"Standard model wheel is not pure py3-none-any: {path.name}")
with zipfile.ZipFile(path) as archive:
names = _validate_zip_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_python_cache(names, path.name)
metadata = _wheel_metadata(archive)
_assert_distribution_metadata(
metadata, STANDARD_NAME, expected_version, path.name
)
if metadata.get("License-Expression") != "CC-BY-SA-3.0":
raise ValueError("Standard model data must declare CC-BY-SA-3.0")
manifest = json.loads(archive.read("radixor_models_standard/manifest.json"))
models = _validate_standard_manifest(manifest, expected_version)
dist_info = next(
name.rsplit("/", 1)[0]
for name in names
if name.endswith(".dist-info/METADATA")
)
allowed = {
"radixor_models_standard/__init__.py",
"radixor_models_standard/manifest.json",
"radixor_models_standard/models/__init__.py",
"radixor_models_standard/notices/__init__.py",
f"{dist_info}/METADATA",
f"{dist_info}/WHEEL",
f"{dist_info}/RECORD",
f"{dist_info}/top_level.txt",
f"{dist_info}/licenses/LICENSE-MODEL-DATA.txt",
}
for model in models:
model_name = f"radixor_models_standard/{model['file']}"
notice_name = f"radixor_models_standard/{model['notice']}"
allowed.update({model_name, notice_name})
data = archive.read(model_name)
if hashlib.sha256(data).hexdigest() != model["sha256"]:
raise ValueError(f"Checksum mismatch for {model['id']}")
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
raise ValueError(f"Invalid v7 marker/version for {model['id']}")
if notice_name not in names:
raise ValueError(f"Missing model notice for {model['id']}")
unexpected = set(names) - allowed
missing = allowed - set(names)
if unexpected or missing:
raise ValueError(
f"Standard wheel allowlist mismatch; unexpected={sorted(unexpected)}, "
f"missing={sorted(missing)}"
)
for info in archive.infolist():
if info.filename in names and ((info.external_attr >> 16) & 0o111):
raise ValueError(
f"Standard wheel contains executable member: {info.filename}"
)
def _verify_standard_sdist(
path: Path, expected_version: str = STANDARD_VERSION
) -> None:
with tarfile.open(path, "r:gz") as archive:
names = _validate_tar_members(archive, path.name)
_assert_no_source_dictionaries(names, path.name)
_assert_no_python_cache(names, path.name)
root = f"radixor_models_standard-{expected_version}"
metadata = BytesParser().parsebytes(
_tar_file_bytes(archive, f"{root}/PKG-INFO")
)
_assert_distribution_metadata(
metadata, STANDARD_NAME, expected_version, path.name
)
manifest_name = f"{root}/radixor_models_standard/manifest.json"
models = _validate_standard_manifest(
json.loads(_tar_file_bytes(archive, manifest_name)), expected_version
)
egg_info = f"{root}/radixor_models_standard.egg-info"
allowed_files = {
f"{root}/LICENSE-MODEL-DATA.txt",
f"{root}/MANIFEST.in",
f"{root}/PKG-INFO",
f"{root}/README.md",
f"{root}/pyproject.toml",
f"{root}/setup.cfg",
f"{root}/radixor_models_standard/__init__.py",
manifest_name,
f"{root}/radixor_models_standard/models/__init__.py",
f"{root}/radixor_models_standard/notices/__init__.py",
f"{egg_info}/PKG-INFO",
f"{egg_info}/SOURCES.txt",
f"{egg_info}/dependency_links.txt",
f"{egg_info}/top_level.txt",
}
allowed_dirs = {
root,
f"{root}/radixor_models_standard",
f"{root}/radixor_models_standard/models",
f"{root}/radixor_models_standard/notices",
egg_info,
}
for model in models:
model_name = f"{root}/radixor_models_standard/{model['file']}"
notice_name = f"{root}/radixor_models_standard/{model['notice']}"
notice_dir = notice_name.rsplit("/", 1)[0]
allowed_files.update({model_name, notice_name})
allowed_dirs.add(notice_dir)
data = _tar_file_bytes(archive, model_name)
if hashlib.sha256(data).hexdigest() != model["sha256"]:
raise ValueError(
f"Checksum mismatch for {model['id']} in standard sdist"
)
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
raise ValueError(
f"Invalid v7 marker/version for {model['id']} in standard sdist"
)
files = {member.name for member in archive.getmembers() if member.isfile()}
directories = {member.name for member in archive.getmembers() if member.isdir()}
if files != allowed_files or directories != allowed_dirs:
raise ValueError(
f"Standard sdist allowlist mismatch; unexpected files={sorted(files - allowed_files)}, "
f"missing files={sorted(allowed_files - files)}, "
f"unexpected dirs={sorted(directories - allowed_dirs)}, "
f"missing dirs={sorted(allowed_dirs - directories)}"
)
executable = [
member.name
for member in archive.getmembers()
if member.isfile() and member.mode & 0o111
]
if executable:
raise ValueError(
f"Standard sdist contains executable members: {executable}"
)
def _verify_fresh_install(main_wheel: Path, standard_wheel: Path) -> None:
managed_temp = REPOSITORY / "build" / "python" / "tmp"
managed_temp.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix="radixor-wheel-install-", dir=managed_temp
) as temporary:
environment = Path(temporary) / "venv"
subprocess.run([sys.executable, "-m", "venv", str(environment)], check=True)
scripts = "Scripts" if os.name == "nt" else "bin"
python = environment / scripts / ("python.exe" if os.name == "nt" else "python")
if "radixor_models_standard-0.0.0-" in standard_wheel.name:
subprocess.run(
[
str(python),
"-m",
"pip",
"install",
"--no-index",
"--no-deps",
str(standard_wheel),
str(main_wheel),
],
check=True,
)
else:
subprocess.run(
[
str(python),
"-m",
"pip",
"install",
"--no-index",
"--find-links",
str(main_wheel.parent),
"--find-links",
str(standard_wheel.parent),
"radixor",
],
check=True,
)
result = subprocess.run(
[
str(python),
"-c",
"from radixor import Stemmer; print(Stemmer('en').stem('running'))",
],
check=True,
capture_output=True,
text=True,
)
if result.stdout.strip() != "run":
raise ValueError(f"Unexpected installed model result: {result.stdout!r}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--main-wheel-dir", type=Path)
parser.add_argument("--main-sdist-dir", type=Path)
parser.add_argument("--standard-dir", type=Path)
parser.add_argument("--main-version", default=MAIN_VERSION)
parser.add_argument("--standard-version", default=STANDARD_VERSION)
parser.add_argument("--skip-install", action="store_true")
args = parser.parse_args()
_assert_no_repository_temp_gzip()
if (args.main_wheel_dir is None) != (args.main_sdist_dir is None):
parser.error("--main-wheel-dir and --main-sdist-dir must be used together")
if args.main_wheel_dir is None and args.standard_dir is None:
parser.error("at least one distribution directory must be provided")
main_wheel = None
standard_wheel = None
if args.main_wheel_dir is not None:
main_wheel = _one(args.main_wheel_dir, "radixor-*.whl")
main_sdist = _one(args.main_sdist_dir, "radixor-*.tar.gz")
_verify_main_wheel(main_wheel, args.main_version)
_verify_main_sdist(main_sdist, args.main_version)
if args.standard_dir is not None:
standard_wheel = _one(args.standard_dir, "radixor_models_standard-*.whl")
standard_sdist = _one(args.standard_dir, "radixor_models_standard-*.tar.gz")
_verify_standard_wheel(standard_wheel, args.standard_version)
_verify_standard_sdist(standard_sdist, args.standard_version)
if not args.skip_install and main_wheel is not None and standard_wheel is not None:
_verify_fresh_install(main_wheel, standard_wheel)
print("verified requested Radixor distributions")
return 0
if __name__ == "__main__":
raise SystemExit(main())