Compare commits

..

4 Commits

14 changed files with 252 additions and 39 deletions

View File

@@ -12,8 +12,8 @@ non-deterministic output; generated payload must never be added to Git.
## Validate without publishing ## Validate without publishing
Run **Python Standard Models Release** manually with version `1.0.0`, then run Run **Python Standard Models Release** manually with version `1.0.0`, then run
**Python Native Release** with version `4.1.0`. `workflow_dispatch` validates **Python Native Release** with version `4.1.0`. The default manual mode validates
artifacts but cannot publish. The native run must pass Linux x86-64, Linux artifacts without publishing. The native run must pass Linux x86-64, Linux
ARM64, macOS universal2, and Windows x86-64. ARM64, macOS universal2, and Windows x86-64.
For the Linux paths, maintainers can use `act` with rootless Podman and the For the Linux paths, maintainers can use `act` with rootless Podman and the

View File

@@ -1,5 +1,6 @@
{ {
"inputs": { "inputs": {
"version": "1.0.0" "version": "1.0.0",
"publish_existing_to_pypi": false
} }
} }

View File

@@ -1,5 +1,6 @@
{ {
"inputs": { "inputs": {
"version": "4.1.0" "version": "4.1.0",
"publish_existing_to_pypi": false
} }
} }

View File

@@ -327,7 +327,10 @@ jobs:
run: | run: |
set -euo pipefail set -euo pipefail
mkdocs build --strict --config-file build/mkdocs/mkdocs.yml mkdocs build --strict --config-file build/mkdocs/mkdocs.yml
rsync -a --delete --exclude '.git' --exclude '.git/' --exclude 'builds/' --exclude 'python/' build/mkdocs-site/ .gh-pages/ rsync -a --delete \
--exclude '.git' --exclude '.git/' \
--exclude 'builds/' --exclude 'python/simple/' \
build/mkdocs-site/ .gh-pages/
mkdir -p .gh-pages/builds mkdir -p .gh-pages/builds
cp build/mkdocs-site/builds/index.html .gh-pages/builds/index.html cp build/mkdocs-site/builds/index.html .gh-pages/builds/index.html
cat > .gh-pages/.nojekyll <<EOF cat > .gh-pages/.nojekyll <<EOF
@@ -337,7 +340,7 @@ jobs:
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
rm -rf .gh-pages/python rm -rf .gh-pages/python/simple
rm -f .gh-pages/.git rm -f .gh-pages/.git
if find .gh-pages -type l -print -quit | grep -q .; then if find .gh-pages -type l -print -quit | grep -q .; then
echo 'Publication candidate contains a symbolic link.' >&2 echo 'Publication candidate contains a symbolic link.' >&2
@@ -391,7 +394,7 @@ jobs:
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
test ! -e candidate/python test ! -e candidate/python/simple
test ! -e candidate/.git test ! -e candidate/.git
if find candidate -type l -print -quit | grep -q .; then if find candidate -type l -print -quit | grep -q .; then
echo 'Publication candidate contains a symbolic link.' >&2 echo 'Publication candidate contains a symbolic link.' >&2
@@ -399,7 +402,8 @@ jobs:
fi fi
rsync -a --delete \ rsync -a --delete \
--exclude '.git' --exclude '.git/' \ --exclude '.git' --exclude '.git/' \
--exclude 'python' --exclude 'python/' candidate/ pages/ --exclude 'python/simple' --exclude 'python/simple/' \
candidate/ pages/
cd pages cd pages
git config user.name "github-actions[bot]" git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

View File

@@ -11,6 +11,11 @@ on:
required: true required: true
default: '1.0.0' default: '1.0.0'
type: string type: string
publish_existing_to_pypi:
description: Publish the existing GitHub Release to PyPI
required: true
default: false
type: boolean
permissions: permissions:
contents: read contents: read
@@ -22,6 +27,7 @@ concurrency:
jobs: jobs:
build: build:
name: Build and verify standard models name: Build and verify standard models
if: github.event_name == 'push' || inputs.publish_existing_to_pypi != true
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
version: ${{ steps.release.outputs.version }} version: ${{ steps.release.outputs.version }}
@@ -201,6 +207,99 @@ jobs:
gh release edit "${RELEASE_TAG}" \ gh release edit "${RELEASE_TAG}" \
--repo "${GITHUB_REPOSITORY}" --draft=false --repo "${GITHUB_REPOSITORY}" --draft=false
publish-pypi:
name: Publish standard models to PyPI
if: >-
always() &&
((github.event_name == 'push' && needs.publish.result == 'success') ||
(github.event_name == 'workflow_dispatch' && inputs.publish_existing_to_pypi))
needs: [build, publish]
runs-on: ubuntu-latest
environment: python-pypi
permissions:
attestations: read
contents: read
id-token: write
steps:
- name: Select PyPI release source
id: release
shell: bash
env:
BUILT_VERSION: ${{ needs.build.outputs.version }}
BUILT_TAG: ${{ needs.build.outputs.tag }}
REQUESTED_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
version="${BUILT_VERSION}"
tag="${BUILT_TAG}"
else
version="${REQUESTED_VERSION}"
tag="python-models-standard@${version}"
fi
[[ "${version}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]
printf 'version=%s\ntag=%s\n' "${version}" "${tag}" >> "${GITHUB_OUTPUT}"
- name: Download current-run release candidate
if: github.event_name == 'push'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: python-models-standard-release-${{ steps.release.outputs.version }}
path: release
- name: Download existing GitHub Release
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
mkdir release
gh release download "${RELEASE_TAG}" \
--repo "${GITHUB_REPOSITORY}" --dir release \
--pattern '*.whl' --pattern '*.tar.gz' --pattern SHA256SUMS
- name: Verify and stage PyPI distributions
shell: bash
run: |
set -euo pipefail
cd release
sha256sum --check SHA256SUMS
awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files
[[ "$(wc -l < expected-files)" -eq 2 ]]
if grep -Ev '^[A-Za-z0-9_.+-]+(\.whl|\.tar\.gz)$' expected-files; then
echo 'SHA256SUMS contains an invalid distribution filename.' >&2
exit 1
fi
find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \
-printf '%f\n' | LC_ALL=C sort > actual-files
diff -u expected-files actual-files
mkdir ../pypi
while IFS= read -r artifact; do
cp -- "${artifact}" ../pypi/
done < expected-files
- name: Verify GitHub build provenance
shell: bash
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
for artifact in pypi/*; do
gh attestation verify "${artifact}" \
--repo "${GITHUB_REPOSITORY}" \
--signer-workflow "${GITHUB_REPOSITORY}/.github/workflows/python-models-standard-release.yml" \
--source-ref "refs/tags/${RELEASE_TAG}"
done
- name: Publish distributions with PyPI Trusted Publishing
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: pypi/
publish-index: publish-index:
name: Publish Python package index name: Publish Python package index
if: github.event_name == 'push' if: github.event_name == 'push'

View File

@@ -11,6 +11,11 @@ on:
required: true required: true
default: '4.1.0' default: '4.1.0'
type: string type: string
publish_existing_to_pypi:
description: Publish the existing GitHub Release to PyPI
required: true
default: false
type: boolean
permissions: permissions:
contents: read contents: read
@@ -22,6 +27,7 @@ concurrency:
jobs: jobs:
prepare: prepare:
name: Prepare versioned sources and sdist name: Prepare versioned sources and sdist
if: github.event_name == 'push' || inputs.publish_existing_to_pypi != true
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
version: ${{ steps.release.outputs.version }} version: ${{ steps.release.outputs.version }}
@@ -419,6 +425,105 @@ jobs:
gh release edit "${RELEASE_TAG}" \ gh release edit "${RELEASE_TAG}" \
--repo "${GITHUB_REPOSITORY}" --draft=false --repo "${GITHUB_REPOSITORY}" --draft=false
publish-pypi:
name: Publish native distributions to PyPI
if: >-
always() &&
((github.event_name == 'push' && needs.publish.result == 'success') ||
(github.event_name == 'workflow_dispatch' && inputs.publish_existing_to_pypi))
needs: [prepare, assemble, publish]
runs-on: ubuntu-latest
environment: python-pypi
permissions:
attestations: read
contents: read
id-token: write
steps:
- name: Select PyPI release source
id: release
shell: bash
env:
BUILT_VERSION: ${{ needs.prepare.outputs.version }}
BUILT_TAG: ${{ needs.prepare.outputs.tag }}
REQUESTED_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
if [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
version="${BUILT_VERSION}"
tag="${BUILT_TAG}"
else
version="${REQUESTED_VERSION}"
tag="python@${version}"
fi
[[ "${version}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]
printf 'version=%s\ntag=%s\n' "${version}" "${tag}" >> "${GITHUB_OUTPUT}"
- name: Download current-run release candidate
if: github.event_name == 'push'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: python-native-release-${{ steps.release.outputs.version }}
path: release
- name: Download existing GitHub Release
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
mkdir release
gh release download "${RELEASE_TAG}" \
--repo "${GITHUB_REPOSITORY}" --dir release \
--pattern '*.whl' --pattern '*.tar.gz' --pattern SHA256SUMS
- name: Verify required model dependency on PyPI
run: >-
curl --fail --silent --show-error
https://pypi.org/pypi/radixor-models-standard/1.0.0/json
> /dev/null
- name: Verify and stage PyPI distributions
shell: bash
run: |
set -euo pipefail
cd release
sha256sum --check SHA256SUMS
awk '{print $2}' SHA256SUMS | LC_ALL=C sort > expected-files
[[ "$(wc -l < expected-files)" -eq 5 ]]
if grep -Ev '^[A-Za-z0-9_.+-]+(\.whl|\.tar\.gz)$' expected-files; then
echo 'SHA256SUMS contains an invalid distribution filename.' >&2
exit 1
fi
find . -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' \) \
-printf '%f\n' | LC_ALL=C sort > actual-files
diff -u expected-files actual-files
mkdir ../pypi
while IFS= read -r artifact; do
cp -- "${artifact}" ../pypi/
done < expected-files
- name: Verify GitHub build provenance
shell: bash
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
run: |
set -euo pipefail
for artifact in pypi/*; do
gh attestation verify "${artifact}" \
--repo "${GITHUB_REPOSITORY}" \
--signer-workflow "${GITHUB_REPOSITORY}/.github/workflows/python-release.yml" \
--source-ref "refs/tags/${RELEASE_TAG}"
done
- name: Publish distributions with PyPI Trusted Publishing
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: pypi/
publish-index: publish-index:
name: Publish Python package index name: Publish Python package index
if: github.event_name == 'push' if: github.event_name == 'push'

View File

@@ -30,7 +30,7 @@ It also retains the operational advantages of a compiled artifact model: predict
For Python, one installation provides the native runtime and the separate For Python, one installation provides the native runtime and the separate
standard package of 20 precompiled models: standard package of 20 precompiled models:
From PyPI, once publication is enabled: From PyPI:
```bash ```bash
python -m pip install --only-binary=:all: radixor python -m pip install --only-binary=:all: radixor

View File

@@ -19,9 +19,8 @@ standard-model package with 20 precompiled language models.
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
``` ```
PyPI publication is pending, and the GitHub index becomes live with the first Both indexes provide the same released distributions. Use PyPI as the primary
Python releases. Until then, follow the source-checkout procedure on source or the GitHub index as an independent alternative.
[Installation and Builds](installation.md).
Radixor supports CPython 3.9 and newer. A JVM, Java dependency, and source Radixor supports CPython 3.9 and newer. A JVM, Java dependency, and source
dictionary are not required. dictionary are not required.

View File

@@ -7,17 +7,12 @@ development or unsupported platforms.
## Install from PyPI ## Install from PyPI
PyPI is the intended primary index once the Radixor projects are approved and PyPI is the primary package index:
published there:
```bash ```bash
python -m pip install --only-binary=:all: radixor python -m pip install --only-binary=:all: radixor
``` ```
PyPI publication is not live yet. Until the `radixor` and
`radixor-models-standard` project pages exist, this command cannot install the
project.
## Install compiled packages from GitHub ## Install compiled packages from GitHub
Python releases are published as immutable GitHub Release assets. A small Python releases are published as immutable GitHub Release assets. A small
@@ -29,10 +24,7 @@ python -m pip install --only-binary=:all: \
``` ```
The index links directly to checksummed wheel assets in GitHub Releases; Pages The index links directly to checksummed wheel assets in GitHub Releases; Pages
does not duplicate the package files. It is not live until the first Python does not duplicate the package files.
model and native releases have been published. This was verified before the
initial release: the URL returned HTTP 404 and the repository contained no
Python Release assets.
Do not configure the GitHub index as an `--extra-index-url`: `pip` does not Do not configure the GitHub index as an `--extra-index-url`: `pip` does not
prioritize one index over another. Use it as the sole `--index-url`, as shown prioritize one index over another. Use it as the sole `--index-url`, as shown
@@ -93,10 +85,8 @@ gh attestation verify radixor-<version>-<wheel-tags>.whl \
Python packages do **not** reuse the OpenPGP key configured for Java/Maven Python packages do **not** reuse the OpenPGP key configured for Java/Maven
Central publications. Java's `SIGNING_KEY` and `SIGNING_PASSWORD` produce Maven Central publications. Java's `SIGNING_KEY` and `SIGNING_PASSWORD` produce Maven
signatures; Python currently uses release checksums plus GitHub's signatures; Python uses release checksums, GitHub identity-bound build
identity-bound build-provenance attestation. A future PyPI publication should provenance, and PyPI Trusted Publishing with supported digital attestations.
use PyPI Trusted Publishing and its supported attestations rather than copying
the Java signing mechanism.
## Build through Gradle ## Build through Gradle

View File

@@ -26,9 +26,8 @@ source .venv/bin/activate # Windows: .venv\Scripts\activate
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
``` ```
PyPI publication is pending. The GitHub option becomes live when the first Both indexes provide the same released distributions. See
Python GitHub Releases populate the Pages-backed package index. See [Installation and Builds](installation.md) for provenance and source builds.
[Installation and Builds](installation.md) for availability and source builds.
The `radixor` wheel contains code. Its required The `radixor` wheel contains code. Its required
`radixor-models-standard` dependency contains 20 precompiled models. The `radixor-models-standard` dependency contains 20 precompiled models. The

View File

@@ -17,12 +17,13 @@ same-input methodology, radixor won all **18 / 18** direct comparisons with
PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run. PyStemmer 3.1.0 (Snowball's C `libstemmer`) in the published 2026-08-08 run.
At batch size 100, the geometric-mean speedup was **1.67×**. The complete At batch size 100, the geometric-mean speedup was **1.67×**. The complete
machine metadata and current results are in the [Python performance machine metadata and current results are in the [Python performance
documentation](../docs/python/performance.md); benchmark implementation and documentation](https://leogalambos.github.io/Radixor/python/performance/);
fairness notes are in [`benchmarks/`](benchmarks/README.md). benchmark implementation and fairness notes are in the
[`benchmarks/` source directory](https://github.com/leogalambos/Radixor/blob/main/python/benchmarks/README.md).
## Installation ## Installation
From PyPI, once publication is enabled: From PyPI:
```bash ```bash
python -m pip install --only-binary=:all: radixor python -m pip install --only-binary=:all: radixor
@@ -35,9 +36,9 @@ python -m pip install --only-binary=:all: \
--index-url https://leogalambos.github.io/Radixor/python/simple/ radixor --index-url https://leogalambos.github.io/Radixor/python/simple/ radixor
``` ```
The GitHub command becomes usable after the first model and native releases The GitHub index points to the same immutable model and native release assets.
populate that index. See the [installation guide](../docs/python/installation.md) See the [installation guide](https://leogalambos.github.io/Radixor/python/installation/)
for current availability and source-checkout builds. for provenance and source-checkout builds.
Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install Wheels are provided for Linux, macOS, and Windows (Python 3.9+). The install
also resolves the mandatory pure `radixor-models-standard` dependency also resolves the mandatory pure `radixor-models-standard` dependency
@@ -249,5 +250,6 @@ individual model versions recorded in the manifest.
## License ## License
The native/API package is BSD-3-Clause — see [LICENSE](LICENSE). Model data The native/API package is BSD-3-Clause — see the
is separately licensed under CC BY-SA 3.0 in its packaged notices. [license text](https://github.com/leogalambos/Radixor/blob/main/python/LICENSE).
Model data is separately licensed under CC BY-SA 3.0 in its packaged notices.

View File

@@ -17,8 +17,9 @@ tag version, and deterministically compiles all model resources there.
Only gzip-compressed Radixor v7 (`.rxc`) tries are shipped. They are release Only gzip-compressed Radixor v7 (`.rxc`) tries are shipped. They are release
artifacts, not checked-in repository files. Canonical textual dictionaries artifacts, not checked-in repository files. Canonical textual dictionaries
remain in the Radixor source repository and are not included in this wheel or remain in the Radixor source repository and are not included in this wheel or
source distribution. Model data is licensed under CC BY-SA 3.0; see source distribution. Model data is licensed under CC BY-SA 3.0; see the
`LICENSE-MODEL-DATA.txt` and the generated per-model notices. [model-data license](https://github.com/leogalambos/Radixor/blob/main/python/models-standard/LICENSE-MODEL-DATA.txt)
and the generated per-model notices.
The checked-out directory is intentionally only a packaging skeleton and is The checked-out directory is intentionally only a packaging skeleton and is
not directly buildable as the complete data distribution. From the repository not directly buildable as the complete data distribution. From the repository

View File

@@ -16,6 +16,12 @@ classifiers = [
"Topic :: Text Processing :: Linguistic", "Topic :: Text Processing :: Linguistic",
] ]
[project.urls]
Homepage = "https://github.com/leogalambos/Radixor"
Documentation = "https://leogalambos.github.io/Radixor/python/"
Repository = "https://github.com/leogalambos/Radixor"
Issues = "https://github.com/leogalambos/Radixor/issues"
[tool.setuptools] [tool.setuptools]
include-package-data = true include-package-data = true

View File

@@ -57,6 +57,12 @@ classifiers = [
"Topic :: Text Processing :: Linguistic", "Topic :: Text Processing :: Linguistic",
] ]
[project.urls]
Homepage = "https://github.com/leogalambos/Radixor"
Documentation = "https://leogalambos.github.io/Radixor/python/"
Repository = "https://github.com/leogalambos/Radixor"
Issues = "https://github.com/leogalambos/Radixor/issues"
[tool.maturin] [tool.maturin]
python-source = "." python-source = "."
module-name = "radixor._radixor" module-name = "radixor._radixor"