feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
This commit is contained in:
171
python/scripts/assemble_release.py
Executable file
171
python/scripts/assemble_release.py
Executable file
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Validate and assemble the exact files allowed in a Python GitHub Release."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from verify_distributions import (
|
||||
_verify_main_sdist,
|
||||
_verify_main_wheel,
|
||||
_verify_standard_sdist,
|
||||
_verify_standard_wheel,
|
||||
)
|
||||
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
BUILD_ROOT = (REPOSITORY / "build").resolve()
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
|
||||
|
||||
|
||||
def _release_files(root: Path) -> list[Path]:
|
||||
files = sorted(
|
||||
path
|
||||
for path in root.rglob("*")
|
||||
if path.is_file()
|
||||
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
|
||||
)
|
||||
names = [path.name for path in files]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError(f"Duplicate release filenames: {names}")
|
||||
return files
|
||||
|
||||
|
||||
def _require_one(files: list[Path], predicate, description: str) -> Path:
|
||||
matches = [path for path in files if predicate(path.name)]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"Expected one {description}, found {[path.name for path in matches]}"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _validate_native(files: list[Path], version: str) -> None:
|
||||
prefix = f"radixor-{version}-cp39-abi3-"
|
||||
wheels = [path for path in files if path.suffix == ".whl"]
|
||||
sdist = _require_one(
|
||||
files, lambda name: name == f"radixor-{version}.tar.gz", "native sdist"
|
||||
)
|
||||
if len(wheels) != 4:
|
||||
raise ValueError(
|
||||
f"Expected four native wheels, found {[path.name for path in wheels]}"
|
||||
)
|
||||
expected = {
|
||||
"linux-x86_64": lambda name: name.startswith(prefix)
|
||||
and "manylinux" in name
|
||||
and name.endswith("x86_64.whl"),
|
||||
"linux-aarch64": lambda name: name.startswith(prefix)
|
||||
and "manylinux" in name
|
||||
and name.endswith("aarch64.whl"),
|
||||
"macos-universal2": lambda name: name.startswith(prefix)
|
||||
and "macosx" in name
|
||||
and name.endswith("universal2.whl"),
|
||||
"windows-x86_64": lambda name: name == f"{prefix}win_amd64.whl",
|
||||
}
|
||||
for description, predicate in expected.items():
|
||||
_require_one(wheels, predicate, description)
|
||||
for wheel in wheels:
|
||||
_verify_main_wheel(wheel, version)
|
||||
_verify_main_sdist(sdist, version)
|
||||
|
||||
|
||||
def _validate_models(files: list[Path], version: str) -> None:
|
||||
wheel = _require_one(
|
||||
files,
|
||||
lambda name: name == f"radixor_models_standard-{version}-py3-none-any.whl",
|
||||
"standard-model wheel",
|
||||
)
|
||||
sdist = _require_one(
|
||||
files,
|
||||
lambda name: name == f"radixor_models_standard-{version}.tar.gz",
|
||||
"standard-model sdist",
|
||||
)
|
||||
if len(files) != 2:
|
||||
raise ValueError(
|
||||
f"Unexpected standard-model release files: {[path.name for path in files]}"
|
||||
)
|
||||
_verify_standard_wheel(wheel, version)
|
||||
_verify_standard_sdist(sdist, version)
|
||||
|
||||
|
||||
def _output_directory(path: Path) -> Path:
|
||||
output = path.resolve()
|
||||
try:
|
||||
output.relative_to(BUILD_ROOT)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Release output must be below {BUILD_ROOT}") from exc
|
||||
if output == BUILD_ROOT:
|
||||
raise ValueError("Release output cannot be the build root")
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
output.mkdir(parents=True)
|
||||
return output
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("distribution", choices=("native", "models-standard"))
|
||||
parser.add_argument("version")
|
||||
parser.add_argument("artifacts", type=Path)
|
||||
parser.add_argument("output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
if VERSION.fullmatch(args.version) is None:
|
||||
raise SystemExit(f"Invalid stable release version: {args.version!r}")
|
||||
files = _release_files(args.artifacts.resolve())
|
||||
if args.distribution == "native":
|
||||
_validate_native(files, args.version)
|
||||
else:
|
||||
_validate_models(files, args.version)
|
||||
|
||||
output = _output_directory(args.output)
|
||||
for source in files:
|
||||
shutil.copy2(source, output / source.name)
|
||||
checksums = [
|
||||
f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}"
|
||||
for path in sorted(output.iterdir())
|
||||
if path.is_file()
|
||||
]
|
||||
(output / "SHA256SUMS").write_text(
|
||||
"\n".join(checksums) + "\n", encoding="utf-8", newline="\n"
|
||||
)
|
||||
print(f"assembled {len(files)} release artifacts in {output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
129
python/scripts/build_standard_distribution.py
Normal file
129
python/scripts/build_standard_distribution.py
Normal file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Build the standard-model wheel and sdist through its declared backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
EXPECTED_MODEL_COUNT = 20
|
||||
|
||||
|
||||
def _ignore_build_artifacts(_directory: str, names: list[str]) -> set[str]:
|
||||
"""Exclude local build state from the isolated distribution source tree."""
|
||||
|
||||
ignored = {
|
||||
name
|
||||
for name in names
|
||||
if name in {"build", "dist", "__pycache__"}
|
||||
or name.endswith((".egg-info", ".pyc", ".pyo"))
|
||||
}
|
||||
return ignored
|
||||
|
||||
|
||||
def _validate_generated_project(project: Path) -> None:
|
||||
"""Reject an ungenerated or incomplete standard-model project."""
|
||||
|
||||
package = project / "radixor_models_standard"
|
||||
manifest_path = package / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise ValueError(
|
||||
"standard-model manifest is missing; run build_standard_models.py first"
|
||||
)
|
||||
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
model_ids = {
|
||||
model["id"] for model in manifest["models"] if isinstance(model, dict)
|
||||
}
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as exc:
|
||||
raise ValueError("standard-model manifest is invalid") from exc
|
||||
|
||||
compiled = {path.stem for path in (package / "models").glob("*.rxc")}
|
||||
notices = {
|
||||
path.parent.name
|
||||
for path in (package / "notices").glob("*/NOTICE-model-data.txt")
|
||||
}
|
||||
if len(model_ids) != EXPECTED_MODEL_COUNT:
|
||||
raise ValueError(
|
||||
f"expected {EXPECTED_MODEL_COUNT} standard models, found {len(model_ids)}"
|
||||
)
|
||||
if compiled != model_ids:
|
||||
raise ValueError("compiled standard models do not match the manifest")
|
||||
if notices != model_ids:
|
||||
raise ValueError("standard-model notices do not match the manifest")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", required=True, type=Path)
|
||||
parser.add_argument("--outdir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
from setuptools.build_meta import build_sdist, build_wheel
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
"setuptools>=77 is required to build radixor-models-standard"
|
||||
) from exc
|
||||
|
||||
source_project = args.project.resolve()
|
||||
try:
|
||||
_validate_generated_project(source_project)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
output = args.outdir.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="radixor-models-build-") as temporary:
|
||||
project = Path(temporary, "project")
|
||||
shutil.copytree(
|
||||
source_project,
|
||||
project,
|
||||
ignore=_ignore_build_artifacts,
|
||||
)
|
||||
os.chdir(project)
|
||||
wheel = build_wheel(str(output))
|
||||
sdist = build_sdist(str(output))
|
||||
|
||||
print(f"built {wheel} and {sdist}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
285
python/scripts/build_standard_models.py
Normal file
285
python/scripts/build_standard_models.py
Normal file
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Regenerate the pure-Python standard model package deterministically.
|
||||
|
||||
The canonical build topology selects ``default`` models. Source dictionaries
|
||||
are compiler inputs only and are never copied into either Python distribution.
|
||||
Run this script with a built Radixor extension importable by the selected
|
||||
Python interpreter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
PYTHON_ROOT = REPOSITORY / "python"
|
||||
BUILD_ROOT = REPOSITORY / "build"
|
||||
SOURCE_PROJECT = PYTHON_ROOT / "models-standard"
|
||||
TOPOLOGY = REPOSITORY / "models" / "model-projects.properties"
|
||||
CATALOG_VERSION = REPOSITORY / "models" / "catalog-version.txt"
|
||||
EXPECTED_FORMAT = {"compression": "gzip", "magic": "EGTR", "version": 7}
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
|
||||
METADATA_FIELDS = (
|
||||
"sourceName",
|
||||
"sourceVersion",
|
||||
"sourceRevision",
|
||||
"sourceProject",
|
||||
"sourceRepository",
|
||||
"sourceDataset",
|
||||
"sourceRevisionStatus",
|
||||
"sourceLicense",
|
||||
"sourceLicenseUri",
|
||||
"sourceAttribution",
|
||||
"sourceVerificationDate",
|
||||
"transformationsSummary",
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _default_models() -> list[str]:
|
||||
entries: dict[str, str] = {}
|
||||
for raw_line in TOPOLOGY.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
model_id, separator, membership = line.partition("=")
|
||||
if not separator or not model_id or membership not in {"default", "optional"}:
|
||||
raise ValueError(f"Invalid model topology line: {raw_line!r}")
|
||||
entries[model_id] = membership
|
||||
model_ids = sorted(
|
||||
model_id for model_id, membership in entries.items() if membership == "default"
|
||||
)
|
||||
if len(model_ids) != 20 or "pl-pl-polimorf" in model_ids:
|
||||
raise ValueError(
|
||||
"Standard Python catalog must contain 20 defaults and exclude pl-pl-polimorf"
|
||||
)
|
||||
return model_ids
|
||||
|
||||
|
||||
def _gradle_metadata(path: Path) -> dict[str, str]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
result: dict[str, str] = {}
|
||||
for field in METADATA_FIELDS:
|
||||
match = re.search(rf"^\s*{field}\s*=\s*'([^']*)'\s*$", text, re.MULTILINE)
|
||||
if match is None:
|
||||
raise ValueError(f"Missing {field} in {path}")
|
||||
result[field] = match.group(1)
|
||||
return result
|
||||
|
||||
|
||||
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
|
||||
if count != 1:
|
||||
raise ValueError(f"Expected exactly one version field in {path}")
|
||||
path.write_text(updated, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def _stage_project(output: Path, distribution_version: str) -> Path:
|
||||
project = output.resolve()
|
||||
try:
|
||||
project.relative_to(BUILD_ROOT.resolve())
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Generated model project must be below {BUILD_ROOT}") from exc
|
||||
if project == BUILD_ROOT.resolve():
|
||||
raise ValueError("Generated model project cannot be the build root")
|
||||
if project.exists():
|
||||
shutil.rmtree(project)
|
||||
project.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(
|
||||
SOURCE_PROJECT,
|
||||
project,
|
||||
ignore=shutil.ignore_patterns(
|
||||
"build",
|
||||
"dist",
|
||||
"*.egg-info",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"manifest.json",
|
||||
"*.rxc",
|
||||
"NOTICE-model-data.txt",
|
||||
),
|
||||
)
|
||||
_replace_once(
|
||||
project / "pyproject.toml",
|
||||
r'^version = "0\.0\.0"$',
|
||||
f'version = "{distribution_version}"',
|
||||
)
|
||||
_replace_once(
|
||||
project / "radixor_models_standard" / "__init__.py",
|
||||
r'^__version__ = "0\.0\.0"$',
|
||||
f'__version__ = "{distribution_version}"',
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def _model_manifest(
|
||||
model_id: str,
|
||||
compile_model: Any,
|
||||
package_root: Path,
|
||||
reproducibility_directory: Path,
|
||||
) -> dict[str, Any]:
|
||||
project = REPOSITORY / "models" / model_id
|
||||
source = project / "src" / "modelInput" / "stemmer.gz"
|
||||
notice = project / "src" / "modelInput" / "NOTICE-model-data.txt"
|
||||
version = (project / "model-version.txt").read_text(encoding="utf-8").strip()
|
||||
if not source.is_file() or not notice.is_file() or not version:
|
||||
raise FileNotFoundError(f"Incomplete canonical model project: {project}")
|
||||
|
||||
destination = package_root / "models" / f"{model_id}.rxc"
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
compile_model(str(source), str(destination), language=model_id)
|
||||
reproduction = reproducibility_directory / destination.name
|
||||
compile_model(str(source), str(reproduction), language=model_id)
|
||||
if destination.read_bytes() != reproduction.read_bytes():
|
||||
raise ValueError(f"Non-deterministic compiled model output for {model_id}")
|
||||
|
||||
notice_destination = package_root / "notices" / model_id / notice.name
|
||||
notice_destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(notice, notice_destination)
|
||||
|
||||
metadata = _gradle_metadata(project / "build.gradle")
|
||||
if metadata["sourceLicense"] != "CC-BY-SA-3.0":
|
||||
raise ValueError(f"Standard model {model_id} must be CC-BY-SA-3.0")
|
||||
return {
|
||||
"file": f"models/{model_id}.rxc",
|
||||
"id": model_id,
|
||||
"notice": f"notices/{model_id}/{notice.name}",
|
||||
"provenance": {
|
||||
"attribution": metadata["sourceAttribution"],
|
||||
"dataset": metadata["sourceDataset"],
|
||||
"license": metadata["sourceLicense"],
|
||||
"license_uri": metadata["sourceLicenseUri"],
|
||||
"repository": metadata["sourceRepository"],
|
||||
"revision": metadata["sourceRevision"],
|
||||
"revision_status": metadata["sourceRevisionStatus"],
|
||||
"source_name": metadata["sourceName"],
|
||||
"source_project": metadata["sourceProject"],
|
||||
"source_version": metadata["sourceVersion"],
|
||||
"transformations": metadata["transformationsSummary"],
|
||||
"verification_date": metadata["sourceVerificationDate"],
|
||||
},
|
||||
"sha256": _sha256(destination),
|
||||
"source": {
|
||||
"path": f"models/{model_id}/src/modelInput/stemmer.gz",
|
||||
"sha256": _sha256(source),
|
||||
},
|
||||
"version": version,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--project", required=True, type=Path)
|
||||
parser.add_argument("--distribution-version", required=True)
|
||||
args = parser.parse_args()
|
||||
if VERSION.fullmatch(args.distribution_version) is None:
|
||||
raise SystemExit(
|
||||
f"Invalid standard-model distribution version: {args.distribution_version!r}"
|
||||
)
|
||||
|
||||
try:
|
||||
from radixor import compile as compile_model
|
||||
except ImportError as exc:
|
||||
print(
|
||||
"error: build the Radixor extension first (for example, "
|
||||
"maturin develop --release) and rerun this script",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(f"detail: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
project = _stage_project(args.project, args.distribution_version)
|
||||
package_root = project / "radixor_models_standard"
|
||||
model_ids = _default_models()
|
||||
expected_files = {f"{model_id}.rxc" for model_id in model_ids}
|
||||
models_directory = package_root / "models"
|
||||
models_directory.mkdir(parents=True, exist_ok=True)
|
||||
for stale in models_directory.glob("*.rxc"):
|
||||
if stale.name not in expected_files:
|
||||
stale.unlink()
|
||||
notices_directory = package_root / "notices"
|
||||
notices_directory.mkdir(parents=True, exist_ok=True)
|
||||
for stale in notices_directory.iterdir():
|
||||
if stale.is_dir() and stale.name not in model_ids:
|
||||
shutil.rmtree(stale)
|
||||
|
||||
reproducibility_root = BUILD_ROOT / "python" / "tmp"
|
||||
reproducibility_root.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="radixor-model-reproducibility-", dir=reproducibility_root
|
||||
) as temporary:
|
||||
models = [
|
||||
_model_manifest(
|
||||
model_id,
|
||||
compile_model,
|
||||
package_root,
|
||||
Path(temporary),
|
||||
)
|
||||
for model_id in model_ids
|
||||
]
|
||||
manifest = {
|
||||
"catalog_version": CATALOG_VERSION.read_text(encoding="utf-8").strip(),
|
||||
"distribution_version": args.distribution_version,
|
||||
"format": EXPECTED_FORMAT,
|
||||
"models": models,
|
||||
"schema_version": 1,
|
||||
"topology": "models/model-projects.properties",
|
||||
}
|
||||
manifest_path = package_root / "manifest.json"
|
||||
with manifest_path.open("w", encoding="utf-8", newline="\n") as stream:
|
||||
stream.write(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
print(f"generated {len(models)} standard compiled models in {package_root}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
132
python/scripts/prepare_release_tree.py
Executable file
132
python/scripts/prepare_release_tree.py
Executable file
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Create an isolated, tag-versioned Python release source tree."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
PYTHON_ROOT = REPOSITORY / "python"
|
||||
BUILD_ROOT = REPOSITORY / "build"
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
|
||||
|
||||
|
||||
def _replace_once(path: Path, pattern: str, replacement: str) -> None:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
updated, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE)
|
||||
if count != 1:
|
||||
raise ValueError(f"Expected exactly one version field in {path}")
|
||||
path.write_text(updated, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def _prepare_output(path: Path) -> Path:
|
||||
output = path.resolve()
|
||||
try:
|
||||
output.relative_to(BUILD_ROOT.resolve())
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Release staging output must be below {BUILD_ROOT}") from exc
|
||||
if output == BUILD_ROOT.resolve():
|
||||
raise ValueError("Release staging output cannot be the build root")
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
return output
|
||||
|
||||
|
||||
def _copy_native(output: Path, version: str) -> None:
|
||||
shutil.copytree(
|
||||
PYTHON_ROOT,
|
||||
output,
|
||||
ignore=shutil.ignore_patterns(
|
||||
".gitignore",
|
||||
".pytest_cache",
|
||||
".ruff_cache",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
"_radixor*.dll",
|
||||
"_radixor*.dylib",
|
||||
"_radixor*.pyd",
|
||||
"_radixor*.so",
|
||||
"_native*.dll",
|
||||
"_native*.dylib",
|
||||
"_native*.pyd",
|
||||
"_native*.so",
|
||||
"benchmarks",
|
||||
"dist",
|
||||
"models",
|
||||
"models-standard",
|
||||
"target",
|
||||
"tests",
|
||||
),
|
||||
)
|
||||
_replace_once(
|
||||
output / "pyproject.toml",
|
||||
r'^version = "0\.0\.0"$',
|
||||
f'version = "{version}"',
|
||||
)
|
||||
_replace_once(
|
||||
output / "Cargo.toml",
|
||||
r'^version = "0\.0\.0"$',
|
||||
f'version = "{version}"',
|
||||
)
|
||||
lock = output / "Cargo.lock"
|
||||
lock_text = lock.read_text(encoding="utf-8")
|
||||
pattern = r'(\[\[package\]\]\nname = "radixor"\n)version = "0\.0\.0"'
|
||||
lock_text, count = re.subn(pattern, rf'\g<1>version = "{version}"', lock_text)
|
||||
if count != 1:
|
||||
raise ValueError("Expected exactly one radixor package entry in Cargo.lock")
|
||||
lock.write_text(lock_text, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("distribution", choices=("native",))
|
||||
parser.add_argument("version")
|
||||
parser.add_argument("output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
if VERSION.fullmatch(args.version) is None:
|
||||
raise SystemExit(f"Invalid stable release version: {args.version!r}")
|
||||
output = _prepare_output(args.output)
|
||||
_copy_native(output, args.version)
|
||||
print(f"prepared {args.distribution} {args.version} in {output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
173
python/scripts/update_simple_index.py
Executable file
173
python/scripts/update_simple_index.py
Executable file
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Update Radixor's static PEP 503 index with verified GitHub Release assets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
PACKAGES = ("radixor", "radixor-models-standard")
|
||||
REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z")
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}\Z")
|
||||
VERSION = re.compile(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\Z")
|
||||
|
||||
|
||||
class _AnchorParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchors: dict[str, str] = {}
|
||||
self._href: str | None = None
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag == "a":
|
||||
self._href = dict(attrs).get("href")
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._href is not None and data.strip():
|
||||
self.anchors[data.strip()] = self._href
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "a":
|
||||
self._href = None
|
||||
|
||||
|
||||
def _existing_links(path: Path, expected_prefix: str) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
parser = _AnchorParser()
|
||||
parser.feed(path.read_text(encoding="utf-8"))
|
||||
for filename, href in parser.anchors.items():
|
||||
base, separator, digest = href.rpartition("#sha256=")
|
||||
if (
|
||||
not separator
|
||||
or not base.startswith(expected_prefix)
|
||||
or SHA256.fullmatch(digest) is None
|
||||
or base.rsplit("/", 1)[-1] != quote(filename, safe="._-")
|
||||
):
|
||||
raise ValueError(f"Existing package-index link is unsafe: {href!r}")
|
||||
return parser.anchors
|
||||
|
||||
|
||||
def _render_root(root: Path) -> None:
|
||||
links = "\n".join(
|
||||
f' <a href="{html.escape(package)}/">{html.escape(package)}</a><br>'
|
||||
for package in PACKAGES
|
||||
)
|
||||
(root / "index.html").write_text(
|
||||
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
|
||||
'content="1.0"><title>Radixor Python packages</title></head>\n'
|
||||
f"<body>\n{links}\n</body></html>\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
|
||||
def _render_project(path: Path, package: str, links: dict[str, str]) -> None:
|
||||
anchors = "\n".join(
|
||||
f' <a href="{html.escape(href, quote=True)}" '
|
||||
f'data-requires-python=">=3.9">{html.escape(filename)}</a><br>'
|
||||
for filename, href in sorted(links.items())
|
||||
)
|
||||
path.write_text(
|
||||
'<!doctype html>\n<html><head><meta name="pypi:repository-version" '
|
||||
f'content="1.0"><title>Links for {html.escape(package)}</title></head>\n'
|
||||
f"<body>\n{anchors}\n</body></html>\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", required=True, type=Path)
|
||||
parser.add_argument("--repository", required=True)
|
||||
parser.add_argument("--package", required=True, choices=PACKAGES)
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--tag", required=True)
|
||||
parser.add_argument("--artifacts", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
if REPOSITORY.fullmatch(args.repository) is None:
|
||||
raise SystemExit("Invalid GitHub repository identity")
|
||||
if VERSION.fullmatch(args.version) is None:
|
||||
raise SystemExit("Invalid stable release version")
|
||||
expected_tag = (
|
||||
f"python@{args.version}"
|
||||
if args.package == "radixor"
|
||||
else f"python-models-standard@{args.version}"
|
||||
)
|
||||
if args.tag != expected_tag:
|
||||
raise SystemExit(f"Tag {args.tag!r} does not match {expected_tag!r}")
|
||||
|
||||
root = args.root.resolve()
|
||||
if root.name != "simple" or root.parent.name != "python":
|
||||
raise SystemExit("The package index must end in python/simple")
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
project = root / args.package
|
||||
project.mkdir(exist_ok=True)
|
||||
page = project / "index.html"
|
||||
prefix = f"https://github.com/{args.repository}/releases/download/"
|
||||
links = _existing_links(page, prefix)
|
||||
release_prefix = f"{prefix}{quote(args.tag, safe='@._-')}/"
|
||||
artifacts = sorted(
|
||||
path
|
||||
for path in args.artifacts.iterdir()
|
||||
if path.is_file()
|
||||
and (path.name.endswith(".whl") or path.name.endswith(".tar.gz"))
|
||||
)
|
||||
if not artifacts:
|
||||
raise SystemExit("No package artifacts were provided")
|
||||
for artifact in artifacts:
|
||||
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
href = f"{release_prefix}{quote(artifact.name, safe='._-')}#sha256={digest}"
|
||||
previous = links.get(artifact.name)
|
||||
if previous is not None and previous != href:
|
||||
raise ValueError(
|
||||
f"Refusing to replace existing index entry {artifact.name}"
|
||||
)
|
||||
links[artifact.name] = href
|
||||
|
||||
_render_root(root)
|
||||
_render_project(page, args.package, links)
|
||||
print(f"indexed {len(artifacts)} artifacts for {args.package} {args.version}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
598
python/scripts/verify_distributions.py
Normal file
598
python/scripts/verify_distributions.py
Normal file
@@ -0,0 +1,598 @@
|
||||
#!/usr/bin/env python3
|
||||
###############################################################################
|
||||
# Copyright (C) 2026, Leo Galambos
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
# may be used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
# POSSIBILITY OF SUCH DAMAGE.
|
||||
###############################################################################
|
||||
|
||||
"""Verify Radixor's native and standard-model release archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from email.parser import BytesParser
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
EXPECTED_DEFAULT_COUNT = 20
|
||||
EXPECTED_DEPENDENCY = "radixor-models-standard>=1.0,<2.0"
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
EXPECTED_NATIVE_LICENSE = REPOSITORY.joinpath("LICENSE").read_bytes()
|
||||
EXPECTED_MODEL_IDS = {
|
||||
"cs-cz-default",
|
||||
"da-dk-default",
|
||||
"de-de-default",
|
||||
"es-es-default",
|
||||
"fa-ir-default",
|
||||
"fi-fi-default",
|
||||
"fr-fr-default",
|
||||
"he-il-default",
|
||||
"hu-hu-default",
|
||||
"it-it-default",
|
||||
"nb-no-default",
|
||||
"nl-nl-default",
|
||||
"nn-no-default",
|
||||
"pl-pl-unimorph",
|
||||
"pt-pt-default",
|
||||
"ru-ru-default",
|
||||
"sv-se-default",
|
||||
"uk-ua-default",
|
||||
"us-uk-default",
|
||||
"yi-default",
|
||||
}
|
||||
STANDARD_NAME = "radixor-models-standard"
|
||||
STANDARD_VERSION = "0.0.0"
|
||||
MAIN_NAME = "radixor"
|
||||
MAIN_VERSION = "0.0.0"
|
||||
|
||||
|
||||
def _one(directory: Path, pattern: str) -> Path:
|
||||
matches = sorted(directory.glob(pattern))
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"Expected exactly one {pattern} in {directory}, found {len(matches)}"
|
||||
)
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _assert_no_repository_temp_gzip() -> None:
|
||||
offenders = sorted(
|
||||
path.name for path in REPOSITORY.glob("tmp*.gz") if path.is_file()
|
||||
)
|
||||
if offenders:
|
||||
raise ValueError(
|
||||
"Repository root contains unmanaged Python temporary gzip files; "
|
||||
f"remove them and fix the producing workflow: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def _wheel_metadata(archive: zipfile.ZipFile) -> object:
|
||||
names = [
|
||||
name for name in archive.namelist() if name.endswith(".dist-info/METADATA")
|
||||
]
|
||||
if len(names) != 1:
|
||||
raise ValueError("Wheel must contain exactly one METADATA file")
|
||||
return BytesParser().parsebytes(archive.read(names[0]))
|
||||
|
||||
|
||||
def _normalized_member_name(name: str, archive_name: str) -> str:
|
||||
if not name or "\\" in name or name.startswith("/"):
|
||||
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
|
||||
stripped = name[:-1] if name.endswith("/") else name
|
||||
parts = stripped.split("/")
|
||||
if not stripped or any(part in {"", ".", ".."} for part in parts):
|
||||
raise ValueError(f"{archive_name} contains unsafe archive path: {name!r}")
|
||||
return "/".join(parts)
|
||||
|
||||
|
||||
def _validate_zip_members(archive: zipfile.ZipFile, archive_name: str) -> list[str]:
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for info in archive.infolist():
|
||||
name = _normalized_member_name(info.filename, archive_name)
|
||||
if name in seen:
|
||||
raise ValueError(f"{archive_name} contains duplicate member: {name}")
|
||||
seen.add(name)
|
||||
mode = (info.external_attr >> 16) & 0xFFFF
|
||||
if stat.S_ISLNK(mode):
|
||||
raise ValueError(f"{archive_name} contains symlink-like ZIP member: {name}")
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def _validate_tar_members(archive: tarfile.TarFile, archive_name: str) -> list[str]:
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for member in archive.getmembers():
|
||||
name = _normalized_member_name(member.name, archive_name)
|
||||
if name in seen:
|
||||
raise ValueError(f"{archive_name} contains duplicate member: {name}")
|
||||
seen.add(name)
|
||||
if not (member.isfile() or member.isdir()):
|
||||
raise ValueError(f"{archive_name} contains unsafe special member: {name}")
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def _assert_distribution_metadata(
|
||||
metadata: object, name: str, version: str, archive_name: str
|
||||
) -> None:
|
||||
if metadata.get("Name") != name or metadata.get("Version") != version:
|
||||
raise ValueError(
|
||||
f"{archive_name} has unexpected distribution identity: "
|
||||
f"{metadata.get('Name')} {metadata.get('Version')}"
|
||||
)
|
||||
|
||||
|
||||
def _tar_file_bytes(archive: tarfile.TarFile, name: str) -> bytes:
|
||||
member = archive.getmember(name)
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None:
|
||||
raise ValueError(f"Cannot read archive member: {name}")
|
||||
return stream.read()
|
||||
|
||||
|
||||
def _validate_standard_manifest(
|
||||
manifest: object, expected_version: str = STANDARD_VERSION
|
||||
) -> list[dict]:
|
||||
if not isinstance(manifest, dict) or set(manifest) != {
|
||||
"catalog_version",
|
||||
"distribution_version",
|
||||
"format",
|
||||
"models",
|
||||
"schema_version",
|
||||
"topology",
|
||||
}:
|
||||
raise ValueError("Standard model manifest has an invalid top-level schema")
|
||||
if (
|
||||
manifest["schema_version"] != 1
|
||||
or manifest["catalog_version"] != "2026.1"
|
||||
or manifest["distribution_version"] != expected_version
|
||||
or manifest["topology"] != "models/model-projects.properties"
|
||||
or manifest["format"] != {"compression": "gzip", "magic": "EGTR", "version": 7}
|
||||
or not isinstance(manifest["models"], list)
|
||||
):
|
||||
raise ValueError("Standard model manifest has incompatible catalog metadata")
|
||||
models = manifest["models"]
|
||||
if {
|
||||
model.get("id") for model in models if isinstance(model, dict)
|
||||
} != EXPECTED_MODEL_IDS:
|
||||
raise ValueError(
|
||||
"Standard model manifest does not match the default model topology"
|
||||
)
|
||||
if len(models) != len(EXPECTED_MODEL_IDS):
|
||||
raise ValueError("Standard model manifest contains duplicate model entries")
|
||||
provenance_keys = {
|
||||
"attribution",
|
||||
"dataset",
|
||||
"license",
|
||||
"license_uri",
|
||||
"repository",
|
||||
"revision",
|
||||
"revision_status",
|
||||
"source_name",
|
||||
"source_project",
|
||||
"source_version",
|
||||
"transformations",
|
||||
"verification_date",
|
||||
}
|
||||
for model in models:
|
||||
if set(model) != {
|
||||
"file",
|
||||
"id",
|
||||
"notice",
|
||||
"provenance",
|
||||
"sha256",
|
||||
"source",
|
||||
"version",
|
||||
}:
|
||||
raise ValueError("Standard model manifest contains an invalid model entry")
|
||||
model_id = model["id"]
|
||||
if (
|
||||
model["file"] != f"models/{model_id}.rxc"
|
||||
or model["notice"] != f"notices/{model_id}/NOTICE-model-data.txt"
|
||||
or model["version"] != "1.0.0"
|
||||
or not isinstance(model["sha256"], str)
|
||||
or len(model["sha256"]) != 64
|
||||
or set(model["provenance"]) != provenance_keys
|
||||
or model["provenance"]["license"] != "CC-BY-SA-3.0"
|
||||
or set(model["source"]) != {"path", "sha256"}
|
||||
or model["source"]["path"] != f"models/{model_id}/src/modelInput/stemmer.gz"
|
||||
):
|
||||
raise ValueError(f"Standard model manifest entry is invalid: {model_id}")
|
||||
if not all(isinstance(value, str) for value in model["provenance"].values()):
|
||||
raise ValueError(f"Standard model provenance is invalid: {model_id}")
|
||||
return models
|
||||
|
||||
|
||||
def _assert_no_source_dictionaries(names: list[str], archive_name: str) -> None:
|
||||
offenders = [
|
||||
name
|
||||
for name in names
|
||||
if name.endswith("stemmer.gz") or "/radixor/models/" in name
|
||||
]
|
||||
if offenders:
|
||||
raise ValueError(
|
||||
f"{archive_name} contains forbidden source dictionaries: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_no_standard_model_payload(names: list[str], archive_name: str) -> None:
|
||||
offenders = []
|
||||
for name in names:
|
||||
parts = PurePosixPath(name).parts
|
||||
if (
|
||||
name.endswith(".rxc")
|
||||
or "models-standard" in parts
|
||||
or any(part.startswith("radixor_models_standard") for part in parts)
|
||||
):
|
||||
offenders.append(name)
|
||||
if offenders:
|
||||
raise ValueError(
|
||||
f"{archive_name} contains standard-model payload owned by "
|
||||
f"radixor-models-standard: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_no_local_build_outputs(names: list[str], archive_name: str) -> None:
|
||||
offenders = []
|
||||
for name in names:
|
||||
path = PurePosixPath(name)
|
||||
parts = path.parts
|
||||
is_benchmark_result = (
|
||||
"benchmarks" in parts
|
||||
and path.name.startswith("results")
|
||||
and path.suffix in {".csv", ".json"}
|
||||
)
|
||||
if "dist" in parts or is_benchmark_result:
|
||||
offenders.append(name)
|
||||
if offenders:
|
||||
raise ValueError(
|
||||
f"{archive_name} contains local distribution or benchmark outputs: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_no_python_cache(
|
||||
names: list[str], archive_name: str, *, allow_native_pyd: bool = False
|
||||
) -> None:
|
||||
offenders = []
|
||||
for name in names:
|
||||
path = PurePosixPath(name)
|
||||
is_native_module = (
|
||||
allow_native_pyd
|
||||
and path.suffix == ".pyd"
|
||||
and path.name.startswith("_radixor")
|
||||
)
|
||||
if (
|
||||
"__pycache__" in path.parts
|
||||
or path.suffix in {".pyc", ".pyo"}
|
||||
or (path.suffix == ".pyd" and not is_native_module)
|
||||
):
|
||||
offenders.append(name)
|
||||
if offenders:
|
||||
raise ValueError(f"{archive_name} contains Python cache artifacts: {offenders}")
|
||||
|
||||
|
||||
def _assert_main_dependency(metadata, archive_name: str) -> None:
|
||||
requirements = metadata.get_all("Requires-Dist", [])
|
||||
normalized = {requirement.replace(" ", "") for requirement in requirements}
|
||||
if EXPECTED_DEPENDENCY not in normalized:
|
||||
raise ValueError(
|
||||
f"Missing compatible standard-model dependency in {archive_name}: "
|
||||
f"{requirements}"
|
||||
)
|
||||
|
||||
|
||||
def _verify_main_wheel(path: Path, expected_version: str = MAIN_VERSION) -> None:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = _validate_zip_members(archive, path.name)
|
||||
_assert_no_source_dictionaries(names, path.name)
|
||||
_assert_no_standard_model_payload(names, path.name)
|
||||
_assert_no_python_cache(names, path.name, allow_native_pyd=True)
|
||||
metadata = _wheel_metadata(archive)
|
||||
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
|
||||
if metadata.get("License-Expression") != "BSD-3-Clause":
|
||||
raise ValueError(
|
||||
f"{path.name} must declare License-Expression: BSD-3-Clause"
|
||||
)
|
||||
license_names = [
|
||||
name for name in names if name.endswith(".dist-info/licenses/LICENSE")
|
||||
]
|
||||
if (
|
||||
len(license_names) != 1
|
||||
or archive.read(license_names[0]) != EXPECTED_NATIVE_LICENSE
|
||||
):
|
||||
raise ValueError(
|
||||
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
|
||||
)
|
||||
_assert_main_dependency(metadata, path.name)
|
||||
|
||||
|
||||
def _verify_main_sdist(path: Path, expected_version: str = MAIN_VERSION) -> None:
|
||||
with tarfile.open(path, "r:gz") as archive:
|
||||
names = _validate_tar_members(archive, path.name)
|
||||
_assert_no_source_dictionaries(names, path.name)
|
||||
_assert_no_standard_model_payload(names, path.name)
|
||||
_assert_no_local_build_outputs(names, path.name)
|
||||
_assert_no_python_cache(names, path.name)
|
||||
license_members = [
|
||||
member
|
||||
for member in archive.getmembers()
|
||||
if PurePosixPath(member.name).name == "LICENSE" and member.isfile()
|
||||
]
|
||||
if len(license_members) != 1:
|
||||
raise ValueError(
|
||||
f"{path.name} must contain exactly one BSD-3-Clause LICENSE"
|
||||
)
|
||||
license_stream = archive.extractfile(license_members[0])
|
||||
if license_stream is None or license_stream.read() != EXPECTED_NATIVE_LICENSE:
|
||||
raise ValueError(
|
||||
f"{path.name} does not contain the full repository BSD-3-Clause LICENSE"
|
||||
)
|
||||
root = f"radixor-{expected_version}"
|
||||
metadata = BytesParser().parsebytes(
|
||||
_tar_file_bytes(archive, f"{root}/PKG-INFO")
|
||||
)
|
||||
_assert_distribution_metadata(metadata, MAIN_NAME, expected_version, path.name)
|
||||
_assert_main_dependency(metadata, path.name)
|
||||
|
||||
|
||||
def _verify_standard_wheel(
|
||||
path: Path, expected_version: str = STANDARD_VERSION
|
||||
) -> None:
|
||||
if not path.name.endswith("-py3-none-any.whl"):
|
||||
raise ValueError(f"Standard model wheel is not pure py3-none-any: {path.name}")
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = _validate_zip_members(archive, path.name)
|
||||
_assert_no_source_dictionaries(names, path.name)
|
||||
_assert_no_python_cache(names, path.name)
|
||||
metadata = _wheel_metadata(archive)
|
||||
_assert_distribution_metadata(
|
||||
metadata, STANDARD_NAME, expected_version, path.name
|
||||
)
|
||||
if metadata.get("License-Expression") != "CC-BY-SA-3.0":
|
||||
raise ValueError("Standard model data must declare CC-BY-SA-3.0")
|
||||
manifest = json.loads(archive.read("radixor_models_standard/manifest.json"))
|
||||
models = _validate_standard_manifest(manifest, expected_version)
|
||||
dist_info = next(
|
||||
name.rsplit("/", 1)[0]
|
||||
for name in names
|
||||
if name.endswith(".dist-info/METADATA")
|
||||
)
|
||||
allowed = {
|
||||
"radixor_models_standard/__init__.py",
|
||||
"radixor_models_standard/manifest.json",
|
||||
"radixor_models_standard/models/__init__.py",
|
||||
"radixor_models_standard/notices/__init__.py",
|
||||
f"{dist_info}/METADATA",
|
||||
f"{dist_info}/WHEEL",
|
||||
f"{dist_info}/RECORD",
|
||||
f"{dist_info}/top_level.txt",
|
||||
f"{dist_info}/licenses/LICENSE-MODEL-DATA.txt",
|
||||
}
|
||||
for model in models:
|
||||
model_name = f"radixor_models_standard/{model['file']}"
|
||||
notice_name = f"radixor_models_standard/{model['notice']}"
|
||||
allowed.update({model_name, notice_name})
|
||||
data = archive.read(model_name)
|
||||
if hashlib.sha256(data).hexdigest() != model["sha256"]:
|
||||
raise ValueError(f"Checksum mismatch for {model['id']}")
|
||||
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
|
||||
raise ValueError(f"Invalid v7 marker/version for {model['id']}")
|
||||
if notice_name not in names:
|
||||
raise ValueError(f"Missing model notice for {model['id']}")
|
||||
unexpected = set(names) - allowed
|
||||
missing = allowed - set(names)
|
||||
if unexpected or missing:
|
||||
raise ValueError(
|
||||
f"Standard wheel allowlist mismatch; unexpected={sorted(unexpected)}, "
|
||||
f"missing={sorted(missing)}"
|
||||
)
|
||||
for info in archive.infolist():
|
||||
if info.filename in names and ((info.external_attr >> 16) & 0o111):
|
||||
raise ValueError(
|
||||
f"Standard wheel contains executable member: {info.filename}"
|
||||
)
|
||||
|
||||
|
||||
def _verify_standard_sdist(
|
||||
path: Path, expected_version: str = STANDARD_VERSION
|
||||
) -> None:
|
||||
with tarfile.open(path, "r:gz") as archive:
|
||||
names = _validate_tar_members(archive, path.name)
|
||||
_assert_no_source_dictionaries(names, path.name)
|
||||
_assert_no_python_cache(names, path.name)
|
||||
root = f"radixor_models_standard-{expected_version}"
|
||||
metadata = BytesParser().parsebytes(
|
||||
_tar_file_bytes(archive, f"{root}/PKG-INFO")
|
||||
)
|
||||
_assert_distribution_metadata(
|
||||
metadata, STANDARD_NAME, expected_version, path.name
|
||||
)
|
||||
manifest_name = f"{root}/radixor_models_standard/manifest.json"
|
||||
models = _validate_standard_manifest(
|
||||
json.loads(_tar_file_bytes(archive, manifest_name)), expected_version
|
||||
)
|
||||
egg_info = f"{root}/radixor_models_standard.egg-info"
|
||||
allowed_files = {
|
||||
f"{root}/LICENSE-MODEL-DATA.txt",
|
||||
f"{root}/MANIFEST.in",
|
||||
f"{root}/PKG-INFO",
|
||||
f"{root}/README.md",
|
||||
f"{root}/pyproject.toml",
|
||||
f"{root}/setup.cfg",
|
||||
f"{root}/radixor_models_standard/__init__.py",
|
||||
manifest_name,
|
||||
f"{root}/radixor_models_standard/models/__init__.py",
|
||||
f"{root}/radixor_models_standard/notices/__init__.py",
|
||||
f"{egg_info}/PKG-INFO",
|
||||
f"{egg_info}/SOURCES.txt",
|
||||
f"{egg_info}/dependency_links.txt",
|
||||
f"{egg_info}/top_level.txt",
|
||||
}
|
||||
allowed_dirs = {
|
||||
root,
|
||||
f"{root}/radixor_models_standard",
|
||||
f"{root}/radixor_models_standard/models",
|
||||
f"{root}/radixor_models_standard/notices",
|
||||
egg_info,
|
||||
}
|
||||
for model in models:
|
||||
model_name = f"{root}/radixor_models_standard/{model['file']}"
|
||||
notice_name = f"{root}/radixor_models_standard/{model['notice']}"
|
||||
notice_dir = notice_name.rsplit("/", 1)[0]
|
||||
allowed_files.update({model_name, notice_name})
|
||||
allowed_dirs.add(notice_dir)
|
||||
data = _tar_file_bytes(archive, model_name)
|
||||
if hashlib.sha256(data).hexdigest() != model["sha256"]:
|
||||
raise ValueError(
|
||||
f"Checksum mismatch for {model['id']} in standard sdist"
|
||||
)
|
||||
if gzip.decompress(data)[:8] != b"EGTR\x00\x00\x00\x07":
|
||||
raise ValueError(
|
||||
f"Invalid v7 marker/version for {model['id']} in standard sdist"
|
||||
)
|
||||
files = {member.name for member in archive.getmembers() if member.isfile()}
|
||||
directories = {member.name for member in archive.getmembers() if member.isdir()}
|
||||
if files != allowed_files or directories != allowed_dirs:
|
||||
raise ValueError(
|
||||
f"Standard sdist allowlist mismatch; unexpected files={sorted(files - allowed_files)}, "
|
||||
f"missing files={sorted(allowed_files - files)}, "
|
||||
f"unexpected dirs={sorted(directories - allowed_dirs)}, "
|
||||
f"missing dirs={sorted(allowed_dirs - directories)}"
|
||||
)
|
||||
executable = [
|
||||
member.name
|
||||
for member in archive.getmembers()
|
||||
if member.isfile() and member.mode & 0o111
|
||||
]
|
||||
if executable:
|
||||
raise ValueError(
|
||||
f"Standard sdist contains executable members: {executable}"
|
||||
)
|
||||
|
||||
|
||||
def _verify_fresh_install(main_wheel: Path, standard_wheel: Path) -> None:
|
||||
managed_temp = REPOSITORY / "build" / "python" / "tmp"
|
||||
managed_temp.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="radixor-wheel-install-", dir=managed_temp
|
||||
) as temporary:
|
||||
environment = Path(temporary) / "venv"
|
||||
subprocess.run([sys.executable, "-m", "venv", str(environment)], check=True)
|
||||
scripts = "Scripts" if os.name == "nt" else "bin"
|
||||
python = environment / scripts / ("python.exe" if os.name == "nt" else "python")
|
||||
if "radixor_models_standard-0.0.0-" in standard_wheel.name:
|
||||
subprocess.run(
|
||||
[
|
||||
str(python),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--no-index",
|
||||
"--no-deps",
|
||||
str(standard_wheel),
|
||||
str(main_wheel),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
subprocess.run(
|
||||
[
|
||||
str(python),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--no-index",
|
||||
"--find-links",
|
||||
str(main_wheel.parent),
|
||||
"--find-links",
|
||||
str(standard_wheel.parent),
|
||||
"radixor",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(python),
|
||||
"-c",
|
||||
"from radixor import Stemmer; print(Stemmer('en').stem('running'))",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.stdout.strip() != "run":
|
||||
raise ValueError(f"Unexpected installed model result: {result.stdout!r}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--main-wheel-dir", type=Path)
|
||||
parser.add_argument("--main-sdist-dir", type=Path)
|
||||
parser.add_argument("--standard-dir", type=Path)
|
||||
parser.add_argument("--main-version", default=MAIN_VERSION)
|
||||
parser.add_argument("--standard-version", default=STANDARD_VERSION)
|
||||
parser.add_argument("--skip-install", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
_assert_no_repository_temp_gzip()
|
||||
if (args.main_wheel_dir is None) != (args.main_sdist_dir is None):
|
||||
parser.error("--main-wheel-dir and --main-sdist-dir must be used together")
|
||||
if args.main_wheel_dir is None and args.standard_dir is None:
|
||||
parser.error("at least one distribution directory must be provided")
|
||||
|
||||
main_wheel = None
|
||||
standard_wheel = None
|
||||
if args.main_wheel_dir is not None:
|
||||
main_wheel = _one(args.main_wheel_dir, "radixor-*.whl")
|
||||
main_sdist = _one(args.main_sdist_dir, "radixor-*.tar.gz")
|
||||
_verify_main_wheel(main_wheel, args.main_version)
|
||||
_verify_main_sdist(main_sdist, args.main_version)
|
||||
if args.standard_dir is not None:
|
||||
standard_wheel = _one(args.standard_dir, "radixor_models_standard-*.whl")
|
||||
standard_sdist = _one(args.standard_dir, "radixor_models_standard-*.tar.gz")
|
||||
_verify_standard_wheel(standard_wheel, args.standard_version)
|
||||
_verify_standard_sdist(standard_sdist, args.standard_version)
|
||||
if not args.skip_install and main_wheel is not None and standard_wheel is not None:
|
||||
_verify_fresh_install(main_wheel, standard_wheel)
|
||||
print("verified requested Radixor distributions")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user