feat(python): add native distribution and release infrastructure
- add the Rust-backed Python API with PyStemmer compatibility - distribute standard compiled models as a separate Python package - generate model artifacts during builds instead of storing them in Git - add GitHub release and Pages-backed package index workflows - add Python tests, benchmarks, documentation, and Gradle integration - refresh the documentation site, branding, and language benchmarks
This commit is contained in:
682
python/src/builder.rs
Normal file
682
python/src/builder.rs
Normal file
@@ -0,0 +1,682 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Port of the Radixor Java trie compilation pipeline
|
||||
// (org.egothor.stemmer.FrequencyTrie.Builder + org.egothor.stemmer.trie.*):
|
||||
// mutable trie build -> bottom-up reduction -> freeze to an immutable compiled trie.
|
||||
//
|
||||
// Faithful port notes:
|
||||
// * Build semantics mirror StemmerPatchTrieLoader.load: for each dictionary
|
||||
// entry we optionally insert the stem mapped to the NOOP patch "Na" (when
|
||||
// store_original) and every variant != stem mapped to
|
||||
// encode_patch(variant, stem, backward).
|
||||
// * Keys are indexed per WordTraversalDirection: BACKWARD consumes characters
|
||||
// right-to-left (logicalIndex = len-1-offset), FORWARD left-to-right.
|
||||
// * Reduction hardcodes the production configuration verified from the Java
|
||||
// source: ReductionMode = MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS,
|
||||
// dominantWinnerMinPercent = 75, dominantWinnerOverSecondRatio = 3,
|
||||
// contractUniformSubtrees = true (metadataForCompilation always applies
|
||||
// ReductionSettings.withUniformSubtreeContraction).
|
||||
// * All character/patch data is handled as UTF-16 code units (Java `char`),
|
||||
// exactly as the runtime trie.rs expects.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::rc::Rc;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::dict::DictEntry;
|
||||
use crate::encoder::encode_patch;
|
||||
use crate::patch::PatchCommand;
|
||||
use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata};
|
||||
|
||||
/// Canonical no-op patch command (PatchCommandEncoder.NOOP_PATCH = "Na").
|
||||
const NOOP_PATCH: &str = "Na";
|
||||
|
||||
/// dominantWinnerMinPercent (ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT).
|
||||
const DOMINANT_WINNER_MIN_PERCENT: i64 = 75;
|
||||
|
||||
/// dominantWinnerOverSecondRatio (ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO).
|
||||
const DOMINANT_WINNER_OVER_SECOND_RATIO: i64 = 3;
|
||||
|
||||
// Ordered value-count map (Java LinkedHashMap<V, Integer> semantics)
|
||||
|
||||
/// Insertion-ordered map from a patch-command string to its accumulated local
|
||||
/// frequency. Mirrors the `LinkedHashMap<V, Integer>` used for `valueCounts` on
|
||||
/// mutable nodes and `localCounts` on reduced nodes.
|
||||
#[derive(Clone, Default)]
|
||||
struct OrderedCounts {
|
||||
entries: Vec<(String, i32)>,
|
||||
index: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
impl OrderedCounts {
|
||||
fn new() -> Self {
|
||||
OrderedCounts {
|
||||
entries: Vec::new(),
|
||||
index: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Adds `count` to `value`, preserving first-seen insertion order. This is
|
||||
/// both the build-time `put` accumulation and the reduction-time
|
||||
/// `mergeLocalCounts` aggregation.
|
||||
fn add(&mut self, value: &str, count: i32) {
|
||||
if let Some(&position) = self.index.get(value) {
|
||||
self.entries[position].1 += count;
|
||||
} else {
|
||||
let position = self.entries.len();
|
||||
self.index.insert(value.to_string(), position);
|
||||
self.entries.push((value.to_string(), count));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MutableNode (org.egothor.stemmer.trie.MutableNode)
|
||||
|
||||
/// Mutable build-time node: children indexed by transition character plus the
|
||||
/// local terminal value counts stored exactly at this node.
|
||||
struct MutableNode {
|
||||
children: BTreeMap<u16, MutableNode>,
|
||||
value_counts: OrderedCounts,
|
||||
}
|
||||
|
||||
impl MutableNode {
|
||||
fn new() -> Self {
|
||||
MutableNode {
|
||||
children: BTreeMap::new(),
|
||||
value_counts: OrderedCounts::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores a value at the node addressed by `key`, incrementing its local
|
||||
/// frequency by one. Mirrors `FrequencyTrie.Builder.put`.
|
||||
fn put(root: &mut MutableNode, key: &[u16], value: &str, backward: bool) {
|
||||
let length = key.len();
|
||||
let mut current = root;
|
||||
for offset in 0..length {
|
||||
// WordTraversalDirection.logicalIndex(length, offset).
|
||||
let logical_index = if backward {
|
||||
length - 1 - offset
|
||||
} else {
|
||||
offset
|
||||
};
|
||||
let edge = key[logical_index];
|
||||
current = current
|
||||
.children
|
||||
.entry(edge)
|
||||
.or_insert_with(MutableNode::new);
|
||||
}
|
||||
current.value_counts.add(value, 1);
|
||||
}
|
||||
|
||||
// ReducedNode (org.egothor.stemmer.trie.ReducedNode)
|
||||
|
||||
/// Canonical reduced node used during subtree merging. Reduced nodes are shared:
|
||||
/// there is exactly one instance per reduction signature, referenced through
|
||||
/// `Rc` so that identical subtrees share a single instance (and therefore a
|
||||
/// single frozen `Arc<CompiledNode>`).
|
||||
struct ReducedNode {
|
||||
/// Canonical reduction signature (see `compute_signature`).
|
||||
signature: String,
|
||||
/// Aggregated local value counts.
|
||||
local_counts: OrderedCounts,
|
||||
/// Canonical children by edge, naturally sorted ascending by the BTreeMap.
|
||||
children: BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
|
||||
/// Whether this node is a contracted accepting leaf.
|
||||
accepts: bool,
|
||||
}
|
||||
|
||||
impl ReducedNode {
|
||||
/// Merges additional local counts into this canonical node.
|
||||
fn merge_local_counts(&mut self, additional: &OrderedCounts) {
|
||||
for (value, count) in &additional.entries {
|
||||
self.local_counts.add(value, *count);
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges child references into this canonical node. For nodes with the same
|
||||
/// reduction signature the child edge sets and child signatures are
|
||||
/// compatible, so this only verifies canonical identity and stores it.
|
||||
fn merge_children(&mut self, additional: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>) {
|
||||
for (edge, child) in additional {
|
||||
match self.children.get(edge) {
|
||||
Some(existing) => {
|
||||
if !Rc::ptr_eq(existing, child) {
|
||||
panic!("Incompatible canonical child encountered during reduction.");
|
||||
}
|
||||
}
|
||||
None => {
|
||||
self.children.insert(*edge, Rc::clone(child));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LocalValueSummary (org.egothor.stemmer.trie.LocalValueSummary)
|
||||
|
||||
/// Deterministic local terminal value summary of a node.
|
||||
struct LocalValueSummary {
|
||||
/// Locally stored values ordered by descending frequency, then shorter text,
|
||||
/// then lexicographic (UTF-16) text, then first-seen insertion order.
|
||||
ordered_values: Vec<String>,
|
||||
/// Frequencies aligned with `ordered_values` (needed for v7 serialization).
|
||||
ordered_counts: Vec<i32>,
|
||||
total_count: i64,
|
||||
dominant_value: Option<String>,
|
||||
dominant_count: i64,
|
||||
second_count: i64,
|
||||
}
|
||||
|
||||
impl LocalValueSummary {
|
||||
/// Builds a summary from local counts, applying the exact Java ordering.
|
||||
fn of(counts: &OrderedCounts) -> Self {
|
||||
struct Sortable {
|
||||
value: String,
|
||||
count: i32,
|
||||
// Java String.length() and String.compareTo operate on UTF-16 code
|
||||
// units, so text ordering must compare the u16 sequence, never UTF-8.
|
||||
text16: Vec<u16>,
|
||||
insertion_order: usize,
|
||||
}
|
||||
|
||||
let mut entries: Vec<Sortable> = counts
|
||||
.entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(insertion_order, (value, count))| Sortable {
|
||||
value: value.clone(),
|
||||
count: *count,
|
||||
text16: value.encode_utf16().collect(),
|
||||
insertion_order,
|
||||
})
|
||||
.collect();
|
||||
|
||||
entries.sort_by(|left, right| {
|
||||
// 1. descending frequency
|
||||
right
|
||||
.count
|
||||
.cmp(&left.count)
|
||||
// 2. shorter text wins
|
||||
.then_with(|| left.text16.len().cmp(&right.text16.len()))
|
||||
// 3. lexicographically lower text (UTF-16 code units) wins
|
||||
.then_with(|| left.text16.cmp(&right.text16))
|
||||
// 4. stable first-seen insertion order
|
||||
.then_with(|| left.insertion_order.cmp(&right.insertion_order))
|
||||
});
|
||||
|
||||
let ordered_values: Vec<String> = entries.iter().map(|entry| entry.value.clone()).collect();
|
||||
let ordered_counts: Vec<i32> = entries.iter().map(|entry| entry.count).collect();
|
||||
let total_count: i64 = entries.iter().map(|entry| entry.count as i64).sum();
|
||||
let dominant_value = entries.first().map(|entry| entry.value.clone());
|
||||
let dominant_count = entries.first().map(|entry| entry.count as i64).unwrap_or(0);
|
||||
let second_count = entries.get(1).map(|entry| entry.count as i64).unwrap_or(0);
|
||||
|
||||
LocalValueSummary {
|
||||
ordered_values,
|
||||
ordered_counts,
|
||||
total_count,
|
||||
dominant_value,
|
||||
dominant_count,
|
||||
second_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the dominant value satisfies both configured dominance
|
||||
/// constraints (percent AND ratio), matching
|
||||
/// `LocalValueSummary.hasQualifiedDominantWinner`.
|
||||
fn has_qualified_dominant_winner(&self) -> bool {
|
||||
if self.dominant_value.is_none() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let percent_satisfied =
|
||||
self.dominant_count * 100 >= self.total_count * DOMINANT_WINNER_MIN_PERCENT;
|
||||
|
||||
let ratio_satisfied = if self.second_count == 0 {
|
||||
true
|
||||
} else {
|
||||
self.dominant_count >= self.second_count * DOMINANT_WINNER_OVER_SECOND_RATIO
|
||||
};
|
||||
|
||||
percent_satisfied && ratio_satisfied
|
||||
}
|
||||
}
|
||||
|
||||
// ReductionSignature (org.egothor.stemmer.trie.ReductionSignature and friends)
|
||||
|
||||
/// Appends `text` to `buffer` using a length-prefixed, collision-free encoding
|
||||
/// so arbitrary UTF-16 patch strings can be embedded without ambiguity.
|
||||
fn push_len_prefixed(buffer: &mut String, text: &str) {
|
||||
buffer.push_str(&text.len().to_string());
|
||||
buffer.push('#');
|
||||
buffer.push_str(text);
|
||||
}
|
||||
|
||||
/// Produces the canonical reduction signature of a subtree as an unambiguous
|
||||
/// hashable string. Two subtrees receive equal signatures exactly when the Java
|
||||
/// `ReductionSignature.equals` would consider them equal:
|
||||
///
|
||||
/// * local descriptor — for DOMINANT mode this is the dominant descriptor
|
||||
/// (only the dominant value) when the summary has a qualified dominant
|
||||
/// winner, otherwise the ranked descriptor (the full ordered value list),
|
||||
/// * whether the node accepts remaining input,
|
||||
/// * the sorted list of (edge label, child signature) pairs.
|
||||
fn compute_signature(
|
||||
summary: &LocalValueSummary,
|
||||
children: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
|
||||
accepts: bool,
|
||||
) -> String {
|
||||
let mut signature = String::new();
|
||||
|
||||
// Local descriptor. 'D' and 'R' markers keep a DominantLocalDescriptor
|
||||
// distinct from a RankedLocalDescriptor holding the same single value,
|
||||
// exactly as the Java class-based equality does.
|
||||
if summary.has_qualified_dominant_winner() {
|
||||
signature.push('D');
|
||||
push_len_prefixed(&mut signature, summary.dominant_value.as_ref().unwrap());
|
||||
} else {
|
||||
signature.push('R');
|
||||
signature.push_str(&summary.ordered_values.len().to_string());
|
||||
signature.push(';');
|
||||
for value in &summary.ordered_values {
|
||||
push_len_prefixed(&mut signature, value);
|
||||
}
|
||||
}
|
||||
|
||||
// acceptsRemainingInput.
|
||||
signature.push(if accepts { 'A' } else { 'a' });
|
||||
|
||||
// Child descriptors in sorted edge order (BTreeMap iterates ascending).
|
||||
signature.push_str(&children.len().to_string());
|
||||
signature.push(';');
|
||||
for (label, child) in children {
|
||||
signature.push_str(&label.to_string());
|
||||
signature.push(':');
|
||||
push_len_prefixed(&mut signature, &child.borrow().signature);
|
||||
}
|
||||
|
||||
signature
|
||||
}
|
||||
|
||||
/// Returns aggregated single-value local counts when the supplied internal
|
||||
/// subtree can be contracted into an accepting leaf, otherwise `None`.
|
||||
///
|
||||
/// Contraction applies (matching `FrequencyTrie.Builder.contractUniformSubtree`)
|
||||
/// when the node has at least one child, every child is a single-value leaf with
|
||||
/// no further children, and all those child values plus the local value (if any)
|
||||
/// are the same single value. The contracted count is always 1.
|
||||
fn contract_uniform_subtree(
|
||||
local_counts: &OrderedCounts,
|
||||
children: &BTreeMap<u16, Rc<RefCell<ReducedNode>>>,
|
||||
) -> Option<OrderedCounts> {
|
||||
if children.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut uniform_value: Option<String> = None;
|
||||
let mut value_seen = false;
|
||||
|
||||
if !local_counts.is_empty() {
|
||||
if local_counts.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
uniform_value = Some(local_counts.entries[0].0.clone());
|
||||
value_seen = true;
|
||||
}
|
||||
|
||||
for child in children.values() {
|
||||
let child_ref = child.borrow();
|
||||
let is_single_value_leaf =
|
||||
child_ref.children.is_empty() && child_ref.local_counts.len() == 1;
|
||||
if !is_single_value_leaf {
|
||||
return None;
|
||||
}
|
||||
let child_value = child_ref.local_counts.entries[0].0.clone();
|
||||
if value_seen && uniform_value.as_deref() != Some(child_value.as_str()) {
|
||||
return None;
|
||||
}
|
||||
uniform_value = Some(child_value);
|
||||
value_seen = true;
|
||||
}
|
||||
|
||||
if !value_seen {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut contracted = OrderedCounts::new();
|
||||
contracted.add(uniform_value.as_ref().unwrap(), 1);
|
||||
Some(contracted)
|
||||
}
|
||||
|
||||
/// Reduces a mutable node to a canonical reduced node (bottom-up).
|
||||
///
|
||||
/// The order of operations mirrors the Java `reduce`:
|
||||
/// 1. reduce every child first,
|
||||
/// 2. try `contractUniformSubtree` (always enabled here),
|
||||
/// 3. compute the local summary and reduction signature,
|
||||
/// 4. deduplicate through the context map, merging counts and children into
|
||||
/// an existing canonical node when the signature already exists.
|
||||
fn reduce(
|
||||
node: &MutableNode,
|
||||
context: &mut HashMap<String, Rc<RefCell<ReducedNode>>>,
|
||||
) -> Rc<RefCell<ReducedNode>> {
|
||||
let mut reduced_children: BTreeMap<u16, Rc<RefCell<ReducedNode>>> = BTreeMap::new();
|
||||
for (edge, child) in node.children.iter() {
|
||||
let reduced_child = reduce(child, context);
|
||||
reduced_children.insert(*edge, reduced_child);
|
||||
}
|
||||
|
||||
let mut local_counts = node.value_counts.clone();
|
||||
let mut accepts_remaining_input = false;
|
||||
|
||||
// contractUniformSubtrees is always true for the production configuration.
|
||||
if let Some(contracted) = contract_uniform_subtree(&local_counts, &reduced_children) {
|
||||
local_counts = contracted;
|
||||
reduced_children = BTreeMap::new();
|
||||
accepts_remaining_input = true;
|
||||
}
|
||||
|
||||
let summary = LocalValueSummary::of(&local_counts);
|
||||
let signature = compute_signature(&summary, &reduced_children, accepts_remaining_input);
|
||||
|
||||
if let Some(canonical) = context.get(&signature).cloned() {
|
||||
{
|
||||
let mut canonical_mut = canonical.borrow_mut();
|
||||
canonical_mut.merge_local_counts(&local_counts);
|
||||
canonical_mut.merge_children(&reduced_children);
|
||||
}
|
||||
return canonical;
|
||||
}
|
||||
|
||||
let canonical = Rc::new(RefCell::new(ReducedNode {
|
||||
signature: signature.clone(),
|
||||
local_counts,
|
||||
children: reduced_children,
|
||||
accepts: accepts_remaining_input,
|
||||
}));
|
||||
context.insert(signature, Rc::clone(&canonical));
|
||||
canonical
|
||||
}
|
||||
|
||||
// Freeze (FrequencyTrie.Builder.freeze -> flat CSR arrays)
|
||||
|
||||
/// Maximum contiguous child-label span for which a node uses a dense
|
||||
/// direct-index table instead of binary search (mirrors the Java
|
||||
/// CompiledNode `maxExpandedIndex` fanout strategy).
|
||||
pub(crate) const MAX_DENSE_SPAN: usize = 512;
|
||||
|
||||
/// Frozen arrays of the compiled trie in CSR layout (see trie.rs).
|
||||
pub(crate) struct FrozenTrie {
|
||||
pub(crate) edge_start: Vec<u32>,
|
||||
pub(crate) edge_labels: Vec<u16>,
|
||||
pub(crate) edge_targets: Vec<u32>,
|
||||
pub(crate) accepts: Vec<bool>,
|
||||
pub(crate) value_start: Vec<u32>,
|
||||
pub(crate) values: Vec<Arc<PatchCommand>>,
|
||||
/// Patch strings parallel to `values` (needed only for serialization).
|
||||
pub(crate) value_strings: Vec<String>,
|
||||
/// Frequencies parallel to `values` (needed only for v7 serialization).
|
||||
pub(crate) value_counts: Vec<i32>,
|
||||
pub(crate) dense_start: Vec<u32>,
|
||||
pub(crate) dense_base: Vec<u16>,
|
||||
pub(crate) dense_targets: Vec<u32>,
|
||||
}
|
||||
|
||||
/// Per-node build record collected during interning, in node-id order.
|
||||
#[derive(Default)]
|
||||
struct NodeBuild {
|
||||
edges: Vec<u16>,
|
||||
targets: Vec<u32>,
|
||||
accepts: bool,
|
||||
values: Vec<Arc<PatchCommand>>,
|
||||
value_strings: Vec<String>,
|
||||
value_counts: Vec<i32>,
|
||||
}
|
||||
|
||||
/// Assigns a stable node id to each distinct canonical reduced node and records
|
||||
/// its edges (ascending), child ids, and best-first values.
|
||||
///
|
||||
/// Shared canonical reduced nodes (identical `Rc` allocations) are interned once
|
||||
/// — the analogue of the Java `IdentityHashMap<ReducedNode, CompiledNode>` cache
|
||||
/// — so structural sharing from reduction is preserved as shared node ids. Equal
|
||||
/// patch strings are compiled once and shared through `patch_cache`.
|
||||
fn intern(
|
||||
node: &Rc<RefCell<ReducedNode>>,
|
||||
index_of: &mut HashMap<usize, u32>,
|
||||
nodes: &mut Vec<NodeBuild>,
|
||||
patch_cache: &mut HashMap<String, Arc<PatchCommand>>,
|
||||
backward: bool,
|
||||
) -> u32 {
|
||||
let identity = Rc::as_ptr(node) as usize;
|
||||
if let Some(&existing) = index_of.get(&identity) {
|
||||
return existing;
|
||||
}
|
||||
let id = nodes.len() as u32;
|
||||
index_of.insert(identity, id);
|
||||
nodes.push(NodeBuild::default()); // reserve this id's slot before recursing
|
||||
|
||||
let node_ref = node.borrow();
|
||||
let summary = LocalValueSummary::of(&node_ref.local_counts);
|
||||
|
||||
// BTreeMap iterates ascending by edge label, so edges stay sorted.
|
||||
let mut edges: Vec<u16> = Vec::with_capacity(node_ref.children.len());
|
||||
let mut targets: Vec<u32> = Vec::with_capacity(node_ref.children.len());
|
||||
for (edge, child) in node_ref.children.iter() {
|
||||
edges.push(*edge);
|
||||
targets.push(intern(child, index_of, nodes, patch_cache, backward));
|
||||
}
|
||||
|
||||
let values: Vec<Arc<PatchCommand>> = summary
|
||||
.ordered_values
|
||||
.iter()
|
||||
.map(|patch| {
|
||||
Arc::clone(
|
||||
patch_cache
|
||||
.entry(patch.clone())
|
||||
.or_insert_with(|| Arc::new(PatchCommand::parse(patch, backward))),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
nodes[id as usize] = NodeBuild {
|
||||
edges,
|
||||
targets,
|
||||
accepts: node_ref.accepts,
|
||||
values,
|
||||
value_strings: summary.ordered_values.clone(),
|
||||
value_counts: summary.ordered_counts.clone(),
|
||||
};
|
||||
id
|
||||
}
|
||||
|
||||
/// Freezes the reduced graph rooted at `root` (node id 0) into flat CSR arrays.
|
||||
fn freeze(root: &Rc<RefCell<ReducedNode>>, backward: bool) -> FrozenTrie {
|
||||
let mut index_of: HashMap<usize, u32> = HashMap::new();
|
||||
let mut nodes: Vec<NodeBuild> = Vec::new();
|
||||
let mut patch_cache: HashMap<String, Arc<PatchCommand>> = HashMap::new();
|
||||
intern(root, &mut index_of, &mut nodes, &mut patch_cache, backward);
|
||||
|
||||
let node_count = nodes.len();
|
||||
let mut edge_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut edge_labels: Vec<u16> = Vec::new();
|
||||
let mut edge_targets: Vec<u32> = Vec::new();
|
||||
let mut accepts: Vec<bool> = Vec::with_capacity(node_count);
|
||||
let mut value_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut values: Vec<Arc<PatchCommand>> = Vec::new();
|
||||
let mut value_strings: Vec<String> = Vec::new();
|
||||
let mut value_counts: Vec<i32> = Vec::new();
|
||||
let mut dense_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut dense_base: Vec<u16> = Vec::with_capacity(node_count);
|
||||
let mut dense_targets: Vec<u32> = Vec::new();
|
||||
|
||||
edge_start.push(0);
|
||||
value_start.push(0);
|
||||
dense_start.push(0);
|
||||
for nb in &nodes {
|
||||
edge_labels.extend_from_slice(&nb.edges);
|
||||
edge_targets.extend_from_slice(&nb.targets);
|
||||
edge_start.push(edge_labels.len() as u32);
|
||||
accepts.push(nb.accepts);
|
||||
for v in &nb.values {
|
||||
values.push(Arc::clone(v));
|
||||
}
|
||||
for v in &nb.value_strings {
|
||||
value_strings.push(v.clone());
|
||||
}
|
||||
value_counts.extend_from_slice(&nb.value_counts);
|
||||
value_start.push(values.len() as u32);
|
||||
|
||||
// Decide dense vs sparse child lookup by fanout/span.
|
||||
let count = nb.edges.len();
|
||||
let mut dense = false;
|
||||
if count >= 2 {
|
||||
let first = nb.edges[0] as usize;
|
||||
let last = nb.edges[count - 1] as usize; // edges are ascending
|
||||
let span = last - first + 1;
|
||||
if span <= MAX_DENSE_SPAN {
|
||||
let base = nb.edges[0];
|
||||
let seg = dense_targets.len();
|
||||
dense_targets.resize(seg + span, 0);
|
||||
for (k, &label) in nb.edges.iter().enumerate() {
|
||||
dense_targets[seg + (label - base) as usize] = nb.targets[k] + 1;
|
||||
}
|
||||
dense_base.push(base);
|
||||
dense_start.push(dense_targets.len() as u32);
|
||||
dense = true;
|
||||
}
|
||||
}
|
||||
if !dense {
|
||||
dense_base.push(0);
|
||||
dense_start.push(dense_targets.len() as u32); // span 0 => sparse
|
||||
}
|
||||
}
|
||||
|
||||
FrozenTrie {
|
||||
edge_start,
|
||||
edge_labels,
|
||||
edge_targets,
|
||||
accepts,
|
||||
value_start,
|
||||
values,
|
||||
value_strings,
|
||||
value_counts,
|
||||
dense_start,
|
||||
dense_base,
|
||||
dense_targets,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metadata_for(backward: bool, lowercase: bool) -> TrieMetadata {
|
||||
TrieMetadata {
|
||||
traversal: if backward {
|
||||
TraversalDirection::Backward
|
||||
} else {
|
||||
TraversalDirection::Forward
|
||||
},
|
||||
case_mode: if lowercase {
|
||||
CaseMode::LowercaseWithLocaleRoot
|
||||
} else {
|
||||
CaseMode::AsIs
|
||||
},
|
||||
diacritic_mode: DiacriticMode::AsIs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the reduced+frozen trie arrays from dictionary entries (shared by the
|
||||
/// in-memory builder and the compiler).
|
||||
pub(crate) fn build_frozen(
|
||||
entries: &[DictEntry],
|
||||
backward: bool,
|
||||
store_original: bool,
|
||||
) -> FrozenTrie {
|
||||
let mut root = MutableNode::new();
|
||||
for entry in entries {
|
||||
let stem16: Vec<u16> = entry.stem.encode_utf16().collect();
|
||||
if store_original {
|
||||
put(&mut root, &stem16, NOOP_PATCH, backward);
|
||||
}
|
||||
for variant in &entry.variants {
|
||||
if variant != &entry.stem {
|
||||
let variant16: Vec<u16> = variant.encode_utf16().collect();
|
||||
let patch = encode_patch(&variant16, &stem16, backward);
|
||||
put(&mut root, &variant16, &patch, backward);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut context: HashMap<String, Rc<RefCell<ReducedNode>>> = HashMap::new();
|
||||
let reduced_root = reduce(&root, &mut context);
|
||||
freeze(&reduced_root, backward)
|
||||
}
|
||||
|
||||
fn frozen_into_trie(frozen: FrozenTrie, metadata: TrieMetadata) -> FrequencyTrie {
|
||||
FrequencyTrie::new(
|
||||
frozen.edge_start,
|
||||
frozen.edge_labels,
|
||||
frozen.edge_targets,
|
||||
frozen.accepts,
|
||||
frozen.value_start,
|
||||
frozen.values,
|
||||
frozen.dense_start,
|
||||
frozen.dense_base,
|
||||
frozen.dense_targets,
|
||||
metadata,
|
||||
)
|
||||
}
|
||||
|
||||
// Public entry point
|
||||
|
||||
/// Compiles dictionary entries into a read-only patch-command trie, faithfully
|
||||
/// reproducing the Java `StemmerPatchTrieLoader.load` build followed by
|
||||
/// `FrequencyTrie.Builder.build` (reduce + freeze).
|
||||
///
|
||||
/// * `backward` — `true` selects BACKWARD traversal (all languages except the
|
||||
/// right-to-left fa/he/yi), `false` selects FORWARD.
|
||||
/// * `store_original` — when `true`, each stem is inserted mapped to the NOOP
|
||||
/// patch `"Na"` so the stem itself is recognised.
|
||||
pub fn build_trie_from_dict(
|
||||
entries: &[DictEntry],
|
||||
backward: bool,
|
||||
store_original: bool,
|
||||
lowercase: bool,
|
||||
) -> FrequencyTrie {
|
||||
let frozen = build_frozen(entries, backward, store_original);
|
||||
frozen_into_trie(frozen, metadata_for(backward, lowercase))
|
||||
}
|
||||
122
python/src/dict.rs
Normal file
122
python/src/dict.rs
Normal file
@@ -0,0 +1,122 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Port of StemmerDictionaryParser (Java) — line-oriented, tab-separated dictionary.
|
||||
//
|
||||
// Layout: first column = canonical stem, following tab-separated columns = variants.
|
||||
// Remarks: the earliest occurrence of `#` or `//` terminates the logical line.
|
||||
// Case: LOWERCASE_WITH_LOCALE_ROOT lowercases the line (locale-independent here).
|
||||
// Items containing any whitespace character are ignored (Java: Character.isWhitespace).
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::{self, Read};
|
||||
|
||||
/// One parsed dictionary entry: a canonical stem and its accepted variants,
|
||||
/// in encounter order.
|
||||
pub struct DictEntry {
|
||||
pub stem: String,
|
||||
pub variants: Vec<String>,
|
||||
}
|
||||
|
||||
/// Decompress gzipped UTF-8 dictionary bytes and parse them into entries.
|
||||
/// `lowercase` mirrors CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT.
|
||||
#[allow(dead_code)] // public helper; the runtime path decompresses then parse_text
|
||||
pub fn parse_gz_dict(compressed: &[u8], lowercase: bool) -> io::Result<Vec<DictEntry>> {
|
||||
let mut decoder = GzDecoder::new(compressed);
|
||||
let mut text = String::new();
|
||||
decoder.read_to_string(&mut text)?;
|
||||
Ok(parse_text(&text, lowercase))
|
||||
}
|
||||
|
||||
/// Parse an already-decompressed dictionary text.
|
||||
pub fn parse_text(text: &str, lowercase: bool) -> Vec<DictEntry> {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
for raw_line in text.lines() {
|
||||
// stripRemark(line).trim(), then lowercase.
|
||||
let stripped = strip_remark(raw_line).trim();
|
||||
if stripped.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let normalized: String = if lowercase {
|
||||
stripped.to_lowercase()
|
||||
} else {
|
||||
stripped.to_string()
|
||||
};
|
||||
if normalized.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// split on '\t' keeping trailing empties (Java split("\t", -1)).
|
||||
let mut columns = normalized.split('\t');
|
||||
|
||||
let stem = match columns.next() {
|
||||
Some(c) => c.trim(),
|
||||
None => continue,
|
||||
};
|
||||
if stem.is_empty() || contains_whitespace(stem) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut variants = Vec::new();
|
||||
for col in columns {
|
||||
let variant = col.trim();
|
||||
if variant.is_empty() || contains_whitespace(variant) {
|
||||
continue;
|
||||
}
|
||||
variants.push(variant.to_string());
|
||||
}
|
||||
|
||||
entries.push(DictEntry {
|
||||
stem: stem.to_string(),
|
||||
variants,
|
||||
});
|
||||
}
|
||||
|
||||
entries
|
||||
}
|
||||
|
||||
/// Removes a trailing remark: the earliest of `#` or `//` terminates the line.
|
||||
fn strip_remark(line: &str) -> &str {
|
||||
let hash = line.find('#');
|
||||
let slash = line.find("//");
|
||||
let remark = match (hash, slash) {
|
||||
(None, None) => return line,
|
||||
(Some(h), None) => h,
|
||||
(None, Some(s)) => s,
|
||||
(Some(h), Some(s)) => h.min(s),
|
||||
};
|
||||
&line[..remark]
|
||||
}
|
||||
|
||||
/// Matches Java Character.isWhitespace closely enough for dictionary items.
|
||||
#[inline]
|
||||
fn contains_whitespace(item: &str) -> bool {
|
||||
item.chars().any(|c| c.is_whitespace())
|
||||
}
|
||||
349
python/src/encoder.rs
Normal file
349
python/src/encoder.rs
Normal file
@@ -0,0 +1,349 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Port of PatchCommandEncoder (Java) — DP-based minimum-cost edit script.
|
||||
// Costs: insert=1, delete=1, replace=1, match=0, mismatch_penalty=100.
|
||||
// Produces compact opcode strings: D(elete), I(nsert), R(eplace), -(skip), N(oop).
|
||||
// Count argument: 'a' + count - 1 (i.e., COUNT_SENTINEL = 'a' - 1 = 96).
|
||||
|
||||
const MISMATCH_PENALTY: i32 = 100;
|
||||
const COUNT_SENTINEL: u16 = b'a' as u16 - 1; // 96 = 0x60
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Trace {
|
||||
Delete,
|
||||
Insert,
|
||||
Replace,
|
||||
Match,
|
||||
}
|
||||
|
||||
/// Encode the patch command that transforms `source` (UTF-16 slice) into `target`.
|
||||
/// Returns "Na" when source == target.
|
||||
pub fn encode_patch(source: &[u16], target: &[u16], backward: bool) -> String {
|
||||
if source == target {
|
||||
return "Na".to_string();
|
||||
}
|
||||
if backward {
|
||||
encode_backward(source, target)
|
||||
} else {
|
||||
encode_forward(source, target)
|
||||
}
|
||||
}
|
||||
|
||||
// Backward traversal encoding.
|
||||
|
||||
fn encode_backward(source: &[u16], target: &[u16]) -> String {
|
||||
let src_len = source.len();
|
||||
let tgt_len = target.len();
|
||||
let cols = tgt_len + 1;
|
||||
|
||||
let mut cost = vec![0i32; (src_len + 1) * cols];
|
||||
let mut trace = vec![Trace::Match; (src_len + 1) * cols];
|
||||
|
||||
let idx = |r: usize, c: usize| r * cols + c;
|
||||
|
||||
// Boundary conditions (Egothor backward: rows=source, cols=target)
|
||||
for i in 1..=src_len {
|
||||
cost[idx(i, 0)] = i as i32;
|
||||
trace[idx(i, 0)] = Trace::Delete;
|
||||
}
|
||||
for j in 1..=tgt_len {
|
||||
cost[idx(0, j)] = j as i32;
|
||||
trace[idx(0, j)] = Trace::Insert;
|
||||
}
|
||||
|
||||
// Fill left-to-right, top-to-bottom (sourceIndex 1..=srcLen, targetIndex 1..=tgtLen)
|
||||
for si in 1..=src_len {
|
||||
let src_ch = source[si - 1]; // sourceCharacters[sourceIndex + sourceCharacterOffset=-1]
|
||||
for ti in 1..=tgt_len {
|
||||
let tgt_ch = target[ti - 1];
|
||||
|
||||
// sourceNeighbor = sourceIndex - 1, targetNeighbor = targetIndex - 1
|
||||
let del = cost[idx(si - 1, ti)] + 1; // DELETE from [si-1][ti]
|
||||
let ins = cost[idx(si, ti - 1)] + 1; // INSERT from [si][ti-1]
|
||||
let diag = cost[idx(si - 1, ti - 1)];
|
||||
let rep = diag + 1;
|
||||
let mat = diag
|
||||
+ if src_ch == tgt_ch {
|
||||
0
|
||||
} else {
|
||||
MISMATCH_PENALTY
|
||||
};
|
||||
|
||||
// Priority: MATCH (baseline), then DELETE (<=), INSERT (<), REPLACE (<)
|
||||
let mut best = mat;
|
||||
let mut bt = Trace::Match;
|
||||
if del <= best {
|
||||
best = del;
|
||||
bt = Trace::Delete;
|
||||
}
|
||||
if ins < best {
|
||||
best = ins;
|
||||
bt = Trace::Insert;
|
||||
}
|
||||
if rep < best {
|
||||
bt = Trace::Replace;
|
||||
}
|
||||
let _ = best;
|
||||
|
||||
cost[idx(si, ti)] = if bt == Trace::Replace {
|
||||
rep
|
||||
} else if bt == Trace::Insert {
|
||||
ins
|
||||
} else if bt == Trace::Delete {
|
||||
del
|
||||
} else {
|
||||
mat
|
||||
};
|
||||
trace[idx(si, ti)] = bt;
|
||||
}
|
||||
}
|
||||
|
||||
build_patch_backward(&trace, target, cols, src_len, tgt_len)
|
||||
}
|
||||
|
||||
fn build_patch_backward(
|
||||
trace: &[Trace],
|
||||
target: &[u16],
|
||||
cols: usize,
|
||||
src_len: usize,
|
||||
tgt_len: usize,
|
||||
) -> String {
|
||||
let idx = |r: usize, c: usize| r * cols + c;
|
||||
|
||||
let mut patch = String::new();
|
||||
let mut pending_deletes: u16 = COUNT_SENTINEL;
|
||||
let mut pending_skips: u16 = COUNT_SENTINEL;
|
||||
|
||||
let mut si = src_len;
|
||||
let mut ti = tgt_len;
|
||||
|
||||
while si != 0 || ti != 0 {
|
||||
match trace[idx(si, ti)] {
|
||||
Trace::Delete => {
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
pending_deletes = pending_deletes.wrapping_add(1);
|
||||
si -= 1;
|
||||
}
|
||||
Trace::Insert => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
ti -= 1;
|
||||
append_instruction(&mut patch, 'I', target[ti]);
|
||||
}
|
||||
Trace::Replace => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
ti -= 1;
|
||||
si -= 1;
|
||||
append_instruction(&mut patch, 'R', target[ti]);
|
||||
}
|
||||
Trace::Match => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
pending_skips = pending_skips.wrapping_add(1);
|
||||
si -= 1;
|
||||
ti -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
}
|
||||
|
||||
patch
|
||||
}
|
||||
|
||||
// Forward traversal encoding.
|
||||
|
||||
fn encode_forward(source: &[u16], target: &[u16]) -> String {
|
||||
let src_len = source.len();
|
||||
let tgt_len = target.len();
|
||||
let cols = tgt_len + 1;
|
||||
|
||||
let mut cost = vec![0i32; (src_len + 1) * cols];
|
||||
let mut trace = vec![Trace::Match; (src_len + 1) * cols];
|
||||
|
||||
let idx = |r: usize, c: usize| r * cols + c;
|
||||
|
||||
// Boundary conditions (fill from bottom-right corner)
|
||||
// cost[srcLen][tgtLen] = 0, trace = MATCH
|
||||
for si in (0..src_len).rev() {
|
||||
cost[idx(si, tgt_len)] = cost[idx(si + 1, tgt_len)] + 1;
|
||||
trace[idx(si, tgt_len)] = Trace::Delete;
|
||||
}
|
||||
for ti in (0..tgt_len).rev() {
|
||||
cost[idx(src_len, ti)] = cost[idx(src_len, ti + 1)] + 1;
|
||||
trace[idx(src_len, ti)] = Trace::Insert;
|
||||
}
|
||||
|
||||
// Fill right-to-left, bottom-to-top
|
||||
for si in (0..src_len).rev() {
|
||||
let src_ch = source[si]; // sourceCharacters[sourceIndex + sourceCharacterOffset=0]
|
||||
for ti in (0..tgt_len).rev() {
|
||||
let tgt_ch = target[ti];
|
||||
|
||||
// sourceNeighbor = sourceIndex + 1, targetNeighbor = targetIndex + 1
|
||||
let del = cost[idx(si + 1, ti)] + 1;
|
||||
let ins = cost[idx(si, ti + 1)] + 1;
|
||||
let diag = cost[idx(si + 1, ti + 1)];
|
||||
let rep = diag + 1;
|
||||
let mat = diag
|
||||
+ if src_ch == tgt_ch {
|
||||
0
|
||||
} else {
|
||||
MISMATCH_PENALTY
|
||||
};
|
||||
|
||||
let mut best = mat;
|
||||
let mut bt = Trace::Match;
|
||||
if del <= best {
|
||||
best = del;
|
||||
bt = Trace::Delete;
|
||||
}
|
||||
if ins < best {
|
||||
best = ins;
|
||||
bt = Trace::Insert;
|
||||
}
|
||||
if rep < best {
|
||||
bt = Trace::Replace;
|
||||
}
|
||||
let _ = best;
|
||||
|
||||
cost[idx(si, ti)] = if bt == Trace::Replace {
|
||||
rep
|
||||
} else if bt == Trace::Insert {
|
||||
ins
|
||||
} else if bt == Trace::Delete {
|
||||
del
|
||||
} else {
|
||||
mat
|
||||
};
|
||||
trace[idx(si, ti)] = bt;
|
||||
}
|
||||
}
|
||||
|
||||
build_patch_forward(&trace, target, cols, src_len, tgt_len)
|
||||
}
|
||||
|
||||
fn build_patch_forward(
|
||||
trace: &[Trace],
|
||||
target: &[u16],
|
||||
cols: usize,
|
||||
src_len: usize,
|
||||
tgt_len: usize,
|
||||
) -> String {
|
||||
let idx = |r: usize, c: usize| r * cols + c;
|
||||
|
||||
let mut patch = String::new();
|
||||
let mut pending_deletes: u16 = COUNT_SENTINEL;
|
||||
let mut pending_skips: u16 = COUNT_SENTINEL;
|
||||
|
||||
let mut si = 0usize;
|
||||
let mut ti = 0usize;
|
||||
|
||||
while si != src_len || ti != tgt_len {
|
||||
match trace[idx(si, ti)] {
|
||||
Trace::Delete => {
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
pending_deletes = pending_deletes.wrapping_add(1);
|
||||
si += 1;
|
||||
}
|
||||
Trace::Insert => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
append_instruction(&mut patch, 'I', target[ti]);
|
||||
ti += 1;
|
||||
}
|
||||
Trace::Replace => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
if pending_skips != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, '-', pending_skips);
|
||||
pending_skips = COUNT_SENTINEL;
|
||||
}
|
||||
append_instruction(&mut patch, 'R', target[ti]);
|
||||
si += 1;
|
||||
ti += 1;
|
||||
}
|
||||
Trace::Match => {
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
pending_deletes = COUNT_SENTINEL;
|
||||
}
|
||||
pending_skips = pending_skips.wrapping_add(1);
|
||||
si += 1;
|
||||
ti += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if pending_deletes != COUNT_SENTINEL {
|
||||
append_instruction(&mut patch, 'D', pending_deletes);
|
||||
}
|
||||
|
||||
patch
|
||||
}
|
||||
|
||||
// Instruction encoding helpers.
|
||||
|
||||
#[inline]
|
||||
fn append_instruction(patch: &mut String, opcode: char, argument: u16) {
|
||||
patch.push(opcode);
|
||||
patch.push(char::from_u32(argument as u32).unwrap_or('\u{FFFD}'));
|
||||
}
|
||||
346
python/src/lib.rs
Normal file
346
python/src/lib.rs
Normal file
@@ -0,0 +1,346 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
mod builder;
|
||||
mod dict;
|
||||
mod encoder;
|
||||
mod patch;
|
||||
mod serial;
|
||||
mod trie;
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::pybacked::PyBackedStr;
|
||||
use pyo3::types::{PyList, PyString};
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use trie::FrequencyTrie;
|
||||
|
||||
/// Decompress a gzip byte image, or return the bytes unchanged when they are
|
||||
/// not gzip-framed (so plain-text dictionaries also work).
|
||||
fn decompress_or_raw(bytes: &[u8]) -> Vec<u8> {
|
||||
if bytes.len() >= 2 && bytes[0] == 0x1F && bytes[1] == 0x8B {
|
||||
let mut out = Vec::new();
|
||||
if GzDecoder::new(bytes).read_to_end(&mut out).is_ok() {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
bytes.to_vec()
|
||||
}
|
||||
|
||||
/// Decode UTF-16 code units into a reused UTF-8 buffer (lossy on unpaired
|
||||
/// surrogates, which never occur in valid patch output).
|
||||
#[inline]
|
||||
fn decode_utf16_into(units: &[u16], out: &mut String) {
|
||||
out.clear();
|
||||
for r in char::decode_utf16(units.iter().copied()) {
|
||||
out.push(r.unwrap_or('\u{FFFD}'));
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime stemmer core: compiles a gzipped textual dictionary into a
|
||||
/// patch-command trie (in Rust) and stems words against it.
|
||||
#[pyclass(module = "radixor._radixor")]
|
||||
struct StemmerCore {
|
||||
trie: Arc<FrequencyTrie>,
|
||||
// Optional result cache (like PyStemmer's): maps an input word to the
|
||||
// already-built Python result object (a str, or None). A hit is a refcount
|
||||
// bump — no re-stemming and no new string. Disabled when `cache_cap == 0`.
|
||||
cache: Option<Mutex<HashMap<String, Py<PyAny>>>>,
|
||||
cache_cap: usize,
|
||||
}
|
||||
|
||||
impl StemmerCore {
|
||||
fn stem_cached(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
word: &str,
|
||||
key_buf: &mut Vec<u16>,
|
||||
u16_buf: &mut Vec<u16>,
|
||||
u8_buf: &mut String,
|
||||
) -> Py<PyAny> {
|
||||
let may_insert = if let Some(cache) = &self.cache {
|
||||
let map = cache.lock().unwrap();
|
||||
if let Some(obj) = map.get(word) {
|
||||
return obj.clone_ref(py);
|
||||
}
|
||||
map.len() < self.cache_cap
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
let computed: Py<PyAny> = match self.trie.stem_len_into(word, key_buf, u16_buf) {
|
||||
Some(_) => {
|
||||
decode_utf16_into(u16_buf, u8_buf);
|
||||
PyString::new_bound(py, u8_buf).into_any().unbind()
|
||||
}
|
||||
None => py.None(),
|
||||
};
|
||||
|
||||
// A full insertion-only cache cannot become writable again, so avoid
|
||||
// a second lock and hash probe for later distinct words.
|
||||
if may_insert {
|
||||
let cache = self.cache.as_ref().expect("enabled cache");
|
||||
let mut map = cache.lock().unwrap();
|
||||
// Another thread may have populated this word while this thread
|
||||
// was stemming it. Return the shared cached object when it did.
|
||||
if let Some(obj) = map.get(word) {
|
||||
return obj.clone_ref(py);
|
||||
}
|
||||
if map.len() < self.cache_cap {
|
||||
map.insert(word.to_owned(), computed.clone_ref(py));
|
||||
}
|
||||
}
|
||||
computed
|
||||
}
|
||||
|
||||
fn stem_batch_impl<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
words: &[PyBackedStr],
|
||||
fallback_to_original: bool,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
let mut key_buf: Vec<u16> = Vec::new();
|
||||
let mut u16_buf: Vec<u16> = Vec::new();
|
||||
let mut u8_buf = String::new();
|
||||
let list = PyList::empty_bound(py);
|
||||
|
||||
// Misses remain cached as None so calls through the compatibility API
|
||||
// cannot change the existing stem/stem_batch missing-value contract.
|
||||
for w in words {
|
||||
let key: &str = w;
|
||||
let obj = self.stem_cached(py, key, &mut key_buf, &mut u16_buf, &mut u8_buf);
|
||||
if fallback_to_original && obj.bind(py).is_none() {
|
||||
list.append(PyString::new_bound(py, key))?;
|
||||
} else {
|
||||
list.append(obj.bind(py))?;
|
||||
}
|
||||
}
|
||||
Ok(list)
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl StemmerCore {
|
||||
/// Compile a model from a gzipped TSV source dictionary.
|
||||
///
|
||||
/// * `path` — path to either a gzipped TSV source dictionary
|
||||
/// (`stem\tvariant1\tvariant2...` per line) OR a compiled `.rxc` trie
|
||||
/// (Java-interoperable v7 format). The format is auto-detected.
|
||||
/// * `backward` — BACKWARD traversal (all languages except the
|
||||
/// right-to-left fa/he/yi, which use FORWARD). Ignored for compiled input
|
||||
/// (baked into the file).
|
||||
/// * `store_original` — map each canonical stem to the no-op patch so the
|
||||
/// stem itself is recognised. Ignored for compiled input.
|
||||
#[new]
|
||||
#[pyo3(signature = (path, backward=true, store_original=true, lowercase=true, cache_size=10_000))]
|
||||
fn new(
|
||||
path: &str,
|
||||
backward: bool,
|
||||
store_original: bool,
|
||||
lowercase: bool,
|
||||
cache_size: usize,
|
||||
) -> PyResult<Self> {
|
||||
let raw =
|
||||
fs::read(path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
|
||||
let decompressed = decompress_or_raw(&raw);
|
||||
// Auto-detect: a compiled v7 trie starts with the stream magic; anything
|
||||
// else is a textual TSV dictionary compiled here in Rust.
|
||||
let trie = if serial::is_v7_stream(&decompressed) {
|
||||
serial::read_stream(&decompressed)
|
||||
.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?
|
||||
} else {
|
||||
// Dictionary keys are always lowercased at build time (canonical
|
||||
// form). `lowercase` controls whether lookups lowercase the input at
|
||||
// runtime; set it False for already-lowercased input.
|
||||
let text = String::from_utf8_lossy(&decompressed);
|
||||
let entries = dict::parse_text(&text, true);
|
||||
builder::build_trie_from_dict(&entries, backward, store_original, lowercase)
|
||||
};
|
||||
let cache = if cache_size > 0 {
|
||||
// Keep PyStemmer's default entry limit without charging every
|
||||
// Stemmer instance for 10,000 buckets before its first lookup.
|
||||
Some(Mutex::new(HashMap::new()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(StemmerCore {
|
||||
trie: Arc::new(trie),
|
||||
cache,
|
||||
cache_cap: cache_size,
|
||||
})
|
||||
}
|
||||
|
||||
fn stem(&self, py: Python<'_>, word: &str) -> Py<PyAny> {
|
||||
self.stem_cached(
|
||||
py,
|
||||
word,
|
||||
&mut Vec::new(),
|
||||
&mut Vec::new(),
|
||||
&mut String::new(),
|
||||
)
|
||||
}
|
||||
|
||||
/// PyStemmer-compatible scalar API. An unrecognized word is returned
|
||||
/// unchanged instead of producing None.
|
||||
#[pyo3(name = "stemWord")]
|
||||
fn stem_word(&self, py: Python<'_>, word: &str) -> Py<PyAny> {
|
||||
let obj = self.stem_cached(
|
||||
py,
|
||||
word,
|
||||
&mut Vec::new(),
|
||||
&mut Vec::new(),
|
||||
&mut String::new(),
|
||||
);
|
||||
if obj.bind(py).is_none() {
|
||||
PyString::new_bound(py, word).into_any().unbind()
|
||||
} else {
|
||||
obj
|
||||
}
|
||||
}
|
||||
|
||||
fn stem_batch<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
words: Vec<PyBackedStr>,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
self.stem_batch_impl(py, &words, false)
|
||||
}
|
||||
|
||||
/// PyStemmer-compatible batch API. Unrecognized words keep their position
|
||||
/// in the result and are returned unchanged.
|
||||
#[pyo3(name = "stemWords")]
|
||||
fn stem_words<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
words: Vec<PyBackedStr>,
|
||||
) -> PyResult<Bound<'py, PyList>> {
|
||||
self.stem_batch_impl(py, &words, true)
|
||||
}
|
||||
|
||||
fn stem_all(&self, word: &str) -> Vec<String> {
|
||||
self.trie.stem_all(word)
|
||||
}
|
||||
|
||||
/// Diagnostic: full batch round-trip (marshal input, allocate one String
|
||||
/// per word, build the result list) with NO stemming. Measures the
|
||||
/// irreducible Python<->Rust boundary + string-allocation floor.
|
||||
fn _echo_batch(&self, words: Vec<PyBackedStr>) -> Vec<Option<String>> {
|
||||
words.iter().map(|w| Some(w.to_string())).collect()
|
||||
}
|
||||
|
||||
/// Diagnostic: pure input marshalling (sum of byte lengths), no stemming,
|
||||
/// no output strings, no result list.
|
||||
fn _len_batch(&self, words: Vec<PyBackedStr>) -> u64 {
|
||||
words.iter().map(|w| w.len() as u64).sum()
|
||||
}
|
||||
|
||||
/// Diagnostic: normalize + UTF-16 encode only.
|
||||
fn _encode_batch(&self, words: Vec<PyBackedStr>) -> u64 {
|
||||
let mut key_buf = Vec::new();
|
||||
words
|
||||
.iter()
|
||||
.map(|w| self.trie.bench_encode(w, &mut key_buf) as u64)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Diagnostic: normalize + encode + trie walk (no patch apply).
|
||||
fn _encodefind_batch(&self, words: Vec<PyBackedStr>) -> u64 {
|
||||
let mut key_buf = Vec::new();
|
||||
let mut acc = 0u64;
|
||||
for w in &words {
|
||||
if self.trie.bench_find(w, &mut key_buf) {
|
||||
acc += 1;
|
||||
}
|
||||
}
|
||||
acc
|
||||
}
|
||||
|
||||
/// Diagnostic: full stemming algorithm (normalize + UTF-16 encode + trie
|
||||
/// walk + patch apply) but returning only the summed stem length — no
|
||||
/// per-word output String and no Python result list.
|
||||
fn _stem_lengths_batch(&self, words: Vec<PyBackedStr>) -> u64 {
|
||||
let mut key_buf = Vec::new();
|
||||
let mut out_buf = Vec::new();
|
||||
let mut acc = 0u64;
|
||||
for w in &words {
|
||||
if let Some(n) = self.trie.stem_len_into(w, &mut key_buf, &mut out_buf) {
|
||||
acc += n as u64;
|
||||
}
|
||||
}
|
||||
acc
|
||||
}
|
||||
|
||||
fn stem_all_batch(&self, words: Vec<PyBackedStr>) -> Vec<Vec<String>> {
|
||||
words.iter().map(|w| self.trie.stem_all(w)).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile a gzipped/plain TSV source dictionary into a Java-interoperable
|
||||
/// compiled trie file (v7 format), so it can be loaded instantly later.
|
||||
///
|
||||
/// * `source_path` — path to a `stemmer.gz` (or plain TSV) source dictionary.
|
||||
/// * `out_path` — destination compiled file (conventionally `*.rxc`).
|
||||
/// * `backward` / `store_original` / `lowercase` — same meaning as the
|
||||
/// `Stemmer` constructor; baked into the compiled file.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (source_path, out_path, backward=true, store_original=true, lowercase=true))]
|
||||
fn compile(
|
||||
source_path: &str,
|
||||
out_path: &str,
|
||||
backward: bool,
|
||||
store_original: bool,
|
||||
lowercase: bool,
|
||||
) -> PyResult<()> {
|
||||
let raw =
|
||||
fs::read(source_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
|
||||
let decompressed = decompress_or_raw(&raw);
|
||||
if serial::is_v7_stream(&decompressed) {
|
||||
return Err(pyo3::exceptions::PyValueError::new_err(
|
||||
"source is already a compiled trie",
|
||||
));
|
||||
}
|
||||
let text = String::from_utf8_lossy(&decompressed);
|
||||
let entries = dict::parse_text(&text, true);
|
||||
let frozen = builder::build_frozen(&entries, backward, store_original);
|
||||
let metadata = builder::metadata_for(backward, lowercase);
|
||||
let bytes = serial::write_v7(&frozen, &metadata)
|
||||
.map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
|
||||
fs::write(out_path, bytes).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[pymodule]
|
||||
fn _radixor(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<StemmerCore>()?;
|
||||
m.add_function(wrap_pyfunction!(compile, m)?)?;
|
||||
Ok(())
|
||||
}
|
||||
494
python/src/patch.rs
Normal file
494
python/src/patch.rs
Normal file
@@ -0,0 +1,494 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PatchCommand {
|
||||
Preserve,
|
||||
DeleteSuffix(usize),
|
||||
DeletePrefix(usize),
|
||||
AppendChar(u16),
|
||||
PrependChar(u16),
|
||||
ReplaceLastChar(u16),
|
||||
ReplaceFirstChar(u16),
|
||||
BackwardCompound {
|
||||
opcodes: Vec<u8>,
|
||||
operands: Vec<u32>,
|
||||
length_delta: i32,
|
||||
min_len: usize,
|
||||
},
|
||||
ForwardCompound {
|
||||
opcodes: Vec<u8>,
|
||||
operands: Vec<u32>,
|
||||
length_delta: i32,
|
||||
min_len: usize,
|
||||
},
|
||||
}
|
||||
|
||||
const SKIP: u8 = b'-';
|
||||
const DELETE: u8 = b'D';
|
||||
const INSERT: u8 = b'I';
|
||||
const REPLACE: u8 = b'R';
|
||||
const NOOP: u8 = b'N';
|
||||
|
||||
fn decode_count(arg: u16) -> Option<usize> {
|
||||
if arg < b'a' as u16 {
|
||||
return None;
|
||||
}
|
||||
Some((arg - b'a' as u16) as usize + 1)
|
||||
}
|
||||
|
||||
fn compile_operand(opcode: u8, arg: u16) -> Option<u32> {
|
||||
match opcode {
|
||||
SKIP | DELETE => {
|
||||
let count = decode_count(arg)?;
|
||||
if count < 1 {
|
||||
None
|
||||
} else {
|
||||
Some(count as u32)
|
||||
}
|
||||
}
|
||||
INSERT | REPLACE => Some(arg as u32),
|
||||
NOOP => {
|
||||
if arg == b'a' as u16 {
|
||||
None
|
||||
} else {
|
||||
panic!("Invalid NOOP arg")
|
||||
}
|
||||
}
|
||||
_ => panic!("Unknown opcode: {}", opcode as char),
|
||||
}
|
||||
}
|
||||
|
||||
fn length_delta(opcodes: &[u8], operands: &[u32]) -> i32 {
|
||||
let mut delta: i32 = 0;
|
||||
for (i, &op) in opcodes.iter().enumerate() {
|
||||
match op {
|
||||
DELETE => delta -= operands[i] as i32,
|
||||
INSERT => delta += 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
delta
|
||||
}
|
||||
|
||||
fn backward_min_len(opcodes: &[u8], operands: &[u32]) -> usize {
|
||||
let mut min_len: usize = 0;
|
||||
let mut consumed_from_end: usize = 0;
|
||||
for (i, &op) in opcodes.iter().enumerate() {
|
||||
let operand = operands[i] as usize;
|
||||
match op {
|
||||
SKIP => consumed_from_end += operand,
|
||||
DELETE => {
|
||||
min_len = min_len.max(consumed_from_end + operand);
|
||||
consumed_from_end += operand;
|
||||
}
|
||||
INSERT => {
|
||||
min_len = min_len.max(consumed_from_end);
|
||||
}
|
||||
REPLACE => {
|
||||
min_len = min_len.max(consumed_from_end + 1);
|
||||
consumed_from_end += 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
min_len
|
||||
}
|
||||
|
||||
fn forward_min_len(opcodes: &[u8], operands: &[u32]) -> usize {
|
||||
let mut min_len: usize = 0;
|
||||
let mut position: i32 = 0;
|
||||
let mut len_delta: i32 = 0;
|
||||
for (i, &op) in opcodes.iter().enumerate() {
|
||||
let operand = operands[i] as i32;
|
||||
match op {
|
||||
SKIP => position += operand,
|
||||
DELETE => {
|
||||
let needed = (position + operand - len_delta).max(0) as usize;
|
||||
min_len = min_len.max(needed);
|
||||
len_delta -= operand;
|
||||
}
|
||||
INSERT => {
|
||||
let needed = (position - len_delta).max(0) as usize;
|
||||
min_len = min_len.max(needed);
|
||||
len_delta += 1;
|
||||
position += 1;
|
||||
}
|
||||
REPLACE => {
|
||||
let needed = (position + 1 - len_delta).max(0) as usize;
|
||||
min_len = min_len.max(needed);
|
||||
position += 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
min_len
|
||||
}
|
||||
|
||||
impl PatchCommand {
|
||||
pub fn parse(patch: &str, backward: bool) -> Self {
|
||||
let chars: Vec<u16> = patch.encode_utf16().collect();
|
||||
let len = chars.len();
|
||||
if len == 0 || len & 1 != 0 {
|
||||
return PatchCommand::Preserve;
|
||||
}
|
||||
|
||||
if len == 2 {
|
||||
let opcode = chars[0] as u8;
|
||||
let arg = chars[1];
|
||||
return Self::compile_single(opcode, arg, backward);
|
||||
}
|
||||
|
||||
let op_count = len / 2;
|
||||
let mut opcodes = Vec::with_capacity(op_count);
|
||||
let mut operands = Vec::with_capacity(op_count);
|
||||
|
||||
for i in 0..op_count {
|
||||
let opcode = chars[i * 2] as u8;
|
||||
let arg = chars[i * 2 + 1];
|
||||
match compile_operand(opcode, arg) {
|
||||
None => return PatchCommand::Preserve,
|
||||
Some(operand) => {
|
||||
opcodes.push(opcode);
|
||||
operands.push(operand);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let ld = length_delta(&opcodes, &operands);
|
||||
if backward {
|
||||
let min_len = backward_min_len(&opcodes, &operands);
|
||||
PatchCommand::BackwardCompound {
|
||||
opcodes,
|
||||
operands,
|
||||
length_delta: ld,
|
||||
min_len,
|
||||
}
|
||||
} else {
|
||||
let min_len = forward_min_len(&opcodes, &operands);
|
||||
PatchCommand::ForwardCompound {
|
||||
opcodes,
|
||||
operands,
|
||||
length_delta: ld,
|
||||
min_len,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compile_single(opcode: u8, arg: u16, backward: bool) -> Self {
|
||||
match opcode {
|
||||
DELETE => {
|
||||
let count = match decode_count(arg) {
|
||||
Some(c) if c >= 1 => c,
|
||||
_ => return PatchCommand::Preserve,
|
||||
};
|
||||
if backward {
|
||||
PatchCommand::DeleteSuffix(count)
|
||||
} else {
|
||||
PatchCommand::DeletePrefix(count)
|
||||
}
|
||||
}
|
||||
INSERT => {
|
||||
if backward {
|
||||
PatchCommand::AppendChar(arg)
|
||||
} else {
|
||||
PatchCommand::PrependChar(arg)
|
||||
}
|
||||
}
|
||||
REPLACE => {
|
||||
if backward {
|
||||
PatchCommand::ReplaceLastChar(arg)
|
||||
} else {
|
||||
PatchCommand::ReplaceFirstChar(arg)
|
||||
}
|
||||
}
|
||||
SKIP | NOOP => PatchCommand::Preserve,
|
||||
_ => panic!("Unknown opcode: {}", opcode as char),
|
||||
}
|
||||
}
|
||||
|
||||
fn computed_length(&self, src_len: usize) -> usize {
|
||||
let (ld, min_len) = match self {
|
||||
PatchCommand::Preserve => (0i32, 0usize),
|
||||
PatchCommand::DeleteSuffix(n) | PatchCommand::DeletePrefix(n) => (-(*n as i32), 0),
|
||||
PatchCommand::AppendChar(_) | PatchCommand::PrependChar(_) => (1, 0),
|
||||
PatchCommand::ReplaceLastChar(_) | PatchCommand::ReplaceFirstChar(_) => (0, 1),
|
||||
PatchCommand::BackwardCompound {
|
||||
length_delta,
|
||||
min_len,
|
||||
..
|
||||
} => (*length_delta, *min_len),
|
||||
PatchCommand::ForwardCompound {
|
||||
length_delta,
|
||||
min_len,
|
||||
..
|
||||
} => (*length_delta, *min_len),
|
||||
};
|
||||
if src_len < min_len {
|
||||
return src_len;
|
||||
}
|
||||
let applied = src_len as i32 + ld;
|
||||
if applied < 1 {
|
||||
src_len
|
||||
} else {
|
||||
applied as usize
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply(&self, source: &[u16]) -> Vec<u16> {
|
||||
let mut out = Vec::new();
|
||||
self.apply_into(source, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Apply the patch into a caller-owned buffer, avoiding a per-call
|
||||
/// allocation on the hot path. `out` is cleared and overwritten.
|
||||
pub fn apply_into(&self, source: &[u16], out: &mut Vec<u16>) {
|
||||
let src_len = source.len();
|
||||
let out_len = self.computed_length(src_len);
|
||||
out.clear();
|
||||
match self {
|
||||
PatchCommand::Preserve => out.extend_from_slice(source),
|
||||
PatchCommand::DeleteSuffix(_) => {
|
||||
if out_len < src_len {
|
||||
out.extend_from_slice(&source[..out_len]);
|
||||
} else {
|
||||
out.extend_from_slice(source);
|
||||
}
|
||||
}
|
||||
PatchCommand::DeletePrefix(n) => {
|
||||
if out_len < src_len {
|
||||
out.extend_from_slice(&source[*n..]);
|
||||
} else {
|
||||
out.extend_from_slice(source);
|
||||
}
|
||||
}
|
||||
PatchCommand::AppendChar(ch) => {
|
||||
out.extend_from_slice(source);
|
||||
out.push(*ch);
|
||||
}
|
||||
PatchCommand::PrependChar(ch) => {
|
||||
out.push(*ch);
|
||||
out.extend_from_slice(source);
|
||||
}
|
||||
PatchCommand::ReplaceLastChar(ch) => {
|
||||
out.extend_from_slice(source);
|
||||
if src_len != 0 {
|
||||
let l = out.len();
|
||||
out[l - 1] = *ch;
|
||||
}
|
||||
}
|
||||
PatchCommand::ReplaceFirstChar(ch) => {
|
||||
out.extend_from_slice(source);
|
||||
if src_len != 0 {
|
||||
out[0] = *ch;
|
||||
}
|
||||
}
|
||||
PatchCommand::BackwardCompound {
|
||||
opcodes, operands, ..
|
||||
} => {
|
||||
if src_len < self.min_len_for_compound() || out_len < 1 {
|
||||
out.extend_from_slice(source);
|
||||
} else {
|
||||
apply_backward_into(opcodes, operands, source, out_len, out);
|
||||
}
|
||||
}
|
||||
PatchCommand::ForwardCompound {
|
||||
opcodes, operands, ..
|
||||
} => {
|
||||
if src_len < self.min_len_for_compound() || out_len < 1 {
|
||||
out.extend_from_slice(source);
|
||||
} else {
|
||||
apply_forward_into(opcodes, operands, source, out_len, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn min_len_for_compound(&self) -> usize {
|
||||
match self {
|
||||
PatchCommand::BackwardCompound { min_len, .. } => *min_len,
|
||||
PatchCommand::ForwardCompound { min_len, .. } => *min_len,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fill_with_source(out: &mut Vec<u16>, source: &[u16]) {
|
||||
out.clear();
|
||||
out.extend_from_slice(source);
|
||||
}
|
||||
|
||||
fn apply_backward_into(
|
||||
opcodes: &[u8],
|
||||
operands: &[u32],
|
||||
source: &[u16],
|
||||
produced_len: usize,
|
||||
out: &mut Vec<u16>,
|
||||
) {
|
||||
let src_len = source.len();
|
||||
out.clear();
|
||||
out.resize(produced_len, 0);
|
||||
let mut current_len = src_len as i32;
|
||||
let mut position = src_len as i32 - 1;
|
||||
let mut src_end = src_len as i32;
|
||||
let mut out_end = produced_len as i32;
|
||||
|
||||
for (i, &op) in opcodes.iter().enumerate() {
|
||||
let operand = operands[i] as i32;
|
||||
match op {
|
||||
SKIP => {
|
||||
let skip = operand.min(src_end);
|
||||
src_end -= skip;
|
||||
out_end -= skip;
|
||||
if out_end < 0 {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
let s = src_end as usize;
|
||||
let o = out_end as usize;
|
||||
out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]);
|
||||
position = position - operand + 1;
|
||||
}
|
||||
DELETE => {
|
||||
let del_end_excl = position + 1;
|
||||
position -= operand - 1;
|
||||
if position < 0 || position > current_len || position > del_end_excl {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
let deleted = (del_end_excl.min(current_len) - position) as i32;
|
||||
if src_end < deleted {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
src_end -= deleted;
|
||||
current_len -= deleted;
|
||||
}
|
||||
INSERT => {
|
||||
if position < -1 || position >= current_len || out_end <= 0 {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
out_end -= 1;
|
||||
out[out_end as usize] = operand as u16;
|
||||
current_len += 1;
|
||||
position += 1;
|
||||
}
|
||||
REPLACE => {
|
||||
if position < 0 || position >= current_len || src_end <= 0 || out_end <= 0 {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
src_end -= 1;
|
||||
out_end -= 1;
|
||||
out[out_end as usize] = operand as u16;
|
||||
}
|
||||
_ => return fill_with_source(out, source),
|
||||
}
|
||||
position -= 1;
|
||||
}
|
||||
|
||||
if src_end != out_end {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
let prefix_len = src_end as usize;
|
||||
out[..prefix_len].copy_from_slice(&source[..prefix_len]);
|
||||
}
|
||||
|
||||
fn apply_forward_into(
|
||||
opcodes: &[u8],
|
||||
operands: &[u32],
|
||||
source: &[u16],
|
||||
produced_len: usize,
|
||||
out: &mut Vec<u16>,
|
||||
) {
|
||||
let src_len = source.len();
|
||||
out.clear();
|
||||
out.resize(produced_len, 0);
|
||||
let mut current_len = src_len as i32;
|
||||
let mut position: i32 = 0;
|
||||
let mut src_idx: i32 = 0;
|
||||
let mut out_idx: i32 = 0;
|
||||
|
||||
for (i, &op) in opcodes.iter().enumerate() {
|
||||
let operand = operands[i] as i32;
|
||||
match op {
|
||||
SKIP => {
|
||||
let skip = operand.min(src_len as i32 - src_idx);
|
||||
let s = src_idx as usize;
|
||||
let o = out_idx as usize;
|
||||
out[o..o + skip as usize].copy_from_slice(&source[s..s + skip as usize]);
|
||||
src_idx += skip;
|
||||
out_idx += skip;
|
||||
position = position + operand - 1;
|
||||
}
|
||||
DELETE => {
|
||||
if position < 0 || position > current_len {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
let del_len = operand.min(current_len - position);
|
||||
if src_idx + del_len > src_len as i32 {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
src_idx += del_len;
|
||||
current_len -= del_len;
|
||||
position -= 1;
|
||||
}
|
||||
INSERT => {
|
||||
if position < 0 || position > current_len || out_idx >= produced_len as i32 {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
out[out_idx as usize] = operand as u16;
|
||||
out_idx += 1;
|
||||
current_len += 1;
|
||||
}
|
||||
REPLACE => {
|
||||
if position < 0
|
||||
|| position >= current_len
|
||||
|| src_idx >= src_len as i32
|
||||
|| out_idx >= produced_len as i32
|
||||
{
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
src_idx += 1;
|
||||
out[out_idx as usize] = operand as u16;
|
||||
out_idx += 1;
|
||||
}
|
||||
_ => return fill_with_source(out, source),
|
||||
}
|
||||
position += 1;
|
||||
}
|
||||
|
||||
let remaining = (src_len as i32 - src_idx) as usize;
|
||||
if remaining > produced_len - out_idx as usize {
|
||||
return fill_with_source(out, source);
|
||||
}
|
||||
let o = out_idx as usize;
|
||||
let s = src_idx as usize;
|
||||
out[o..o + remaining].copy_from_slice(&source[s..s + remaining]);
|
||||
if out_idx as usize + remaining != produced_len {
|
||||
fill_with_source(out, source);
|
||||
}
|
||||
}
|
||||
450
python/src/serial.rs
Normal file
450
python/src/serial.rs
Normal file
@@ -0,0 +1,450 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Java-interoperable compiled-trie binary I/O ("v7" stream), matching
|
||||
// org.egothor.stemmer.StemmerPatchTrieBinaryIO / FrequencyTrie.writeTo/readFrom.
|
||||
//
|
||||
// File layout = gzip( big-endian Java DataOutputStream stream ):
|
||||
// i32 STREAM_MAGIC=0x45475452 ; i32 STREAM_VERSION=7
|
||||
// i32 nodeCount ; i32 rootId(=0)
|
||||
// writeUTF(metadata.toTextBlock()) // Java modified UTF-8
|
||||
// i32 valueCount ; valueCount x writeUTF(patch) // value dictionary
|
||||
// per node id 0..nodeCount-1:
|
||||
// u8 acceptsRemainingInput
|
||||
// i32 edgeCount ; edgeCount x { u16 edgeLabel ; i32 childId }
|
||||
// i32 valueCount ; valueCount x { i32 valueId ; i32 count }
|
||||
//
|
||||
// The outer gzip framing (headers/mtime) is not byte-identical across Java and
|
||||
// Rust, but the INNER stream is, and both directions gunzip+parse each other.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::sync::Arc;
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use flate2::write::GzEncoder;
|
||||
use flate2::Compression;
|
||||
|
||||
use crate::builder::{FrozenTrie, MAX_DENSE_SPAN};
|
||||
use crate::patch::PatchCommand;
|
||||
use crate::trie::{CaseMode, DiacriticMode, FrequencyTrie, TraversalDirection, TrieMetadata};
|
||||
|
||||
const STREAM_MAGIC: i32 = 0x4547_5452;
|
||||
const STREAM_VERSION: i32 = 7;
|
||||
|
||||
// Big-endian writer helpers matching Java DataOutputStream.
|
||||
|
||||
fn put_i32(out: &mut Vec<u8>, v: i32) {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
|
||||
fn put_u16(out: &mut Vec<u8>, v: u16) {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
|
||||
/// Java DataOutputStream.writeUTF: u16 big-endian byte length + modified UTF-8.
|
||||
fn put_java_utf(out: &mut Vec<u8>, s: &str) -> io::Result<()> {
|
||||
let mut bytes: Vec<u8> = Vec::with_capacity(s.len());
|
||||
for u in s.encode_utf16() {
|
||||
if (0x0001..=0x007F).contains(&u) {
|
||||
bytes.push(u as u8);
|
||||
} else if u == 0 || (0x0080..=0x07FF).contains(&u) {
|
||||
bytes.push(0xC0 | ((u >> 6) as u8 & 0x1F));
|
||||
bytes.push(0x80 | (u as u8 & 0x3F));
|
||||
} else {
|
||||
bytes.push(0xE0 | ((u >> 12) as u8 & 0x0F));
|
||||
bytes.push(0x80 | ((u >> 6) as u8 & 0x3F));
|
||||
bytes.push(0x80 | (u as u8 & 0x3F));
|
||||
}
|
||||
}
|
||||
if bytes.len() > 0xFFFF {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"string too long for Java modified UTF-8",
|
||||
));
|
||||
}
|
||||
put_u16(out, bytes.len() as u16);
|
||||
out.extend_from_slice(&bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Metadata text block, byte-identical to TrieMetadata.toTextBlock.
|
||||
|
||||
fn text_block(meta: &TrieMetadata) -> String {
|
||||
let forward = matches!(meta.traversal, TraversalDirection::Forward);
|
||||
let case = match meta.case_mode {
|
||||
CaseMode::LowercaseWithLocaleRoot => "LOWERCASE_WITH_LOCALE_ROOT",
|
||||
CaseMode::AsIs => "AS_IS",
|
||||
};
|
||||
let diac = match meta.diacritic_mode {
|
||||
DiacriticMode::AsIs => "AS_IS",
|
||||
DiacriticMode::Remove => "REMOVE",
|
||||
};
|
||||
let mut s = String::with_capacity(256);
|
||||
s.push_str("radixor.metadata.v1\n");
|
||||
s.push_str("formatVersion=7\n");
|
||||
s.push_str(if forward {
|
||||
"traversalDirection=FORWARD\n"
|
||||
} else {
|
||||
"traversalDirection=BACKWARD\n"
|
||||
});
|
||||
s.push_str(if forward {
|
||||
"rightToLeft=true\n"
|
||||
} else {
|
||||
"rightToLeft=false\n"
|
||||
});
|
||||
s.push_str("reductionMode=MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS\n");
|
||||
s.push_str("dominantWinnerMinPercent=75\n");
|
||||
s.push_str("dominantWinnerOverSecondRatio=3\n");
|
||||
s.push_str("contractUniformSubtrees=true\n");
|
||||
s.push_str(&format!("diacriticProcessingMode={}\n", diac));
|
||||
s.push_str(&format!("caseProcessingMode={}\n", case));
|
||||
s
|
||||
}
|
||||
|
||||
/// Serialize the frozen trie to the inner (uncompressed) v7 stream.
|
||||
fn write_stream(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result<Vec<u8>> {
|
||||
let node_count = frozen.accepts.len();
|
||||
let mut out = Vec::with_capacity(1024 + frozen.edge_labels.len() * 6);
|
||||
|
||||
put_i32(&mut out, STREAM_MAGIC);
|
||||
put_i32(&mut out, STREAM_VERSION);
|
||||
put_i32(&mut out, node_count as i32);
|
||||
put_i32(&mut out, 0); // rootId
|
||||
put_java_utf(&mut out, &text_block(meta))?;
|
||||
|
||||
// Value dictionary: distinct patch strings in first-occurrence order across
|
||||
// nodes(id) x values(local) — frozen.value_strings is already in that order.
|
||||
let mut value_id: HashMap<&str, i32> = HashMap::new();
|
||||
let mut distinct: Vec<&str> = Vec::new();
|
||||
for s in &frozen.value_strings {
|
||||
if !value_id.contains_key(s.as_str()) {
|
||||
value_id.insert(s.as_str(), distinct.len() as i32);
|
||||
distinct.push(s.as_str());
|
||||
}
|
||||
}
|
||||
put_i32(&mut out, distinct.len() as i32);
|
||||
for s in &distinct {
|
||||
put_java_utf(&mut out, s)?;
|
||||
}
|
||||
|
||||
for node in 0..node_count {
|
||||
out.push(if frozen.accepts[node] { 1 } else { 0 });
|
||||
|
||||
let elo = frozen.edge_start[node] as usize;
|
||||
let ehi = frozen.edge_start[node + 1] as usize;
|
||||
put_i32(&mut out, (ehi - elo) as i32);
|
||||
for k in elo..ehi {
|
||||
put_u16(&mut out, frozen.edge_labels[k]);
|
||||
put_i32(&mut out, frozen.edge_targets[k] as i32);
|
||||
}
|
||||
|
||||
let vlo = frozen.value_start[node] as usize;
|
||||
let vhi = frozen.value_start[node + 1] as usize;
|
||||
put_i32(&mut out, (vhi - vlo) as i32);
|
||||
for k in vlo..vhi {
|
||||
let id = value_id[frozen.value_strings[k].as_str()];
|
||||
put_i32(&mut out, id);
|
||||
put_i32(&mut out, frozen.value_counts[k]);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Serialize the frozen trie to a gzip-compressed v7 file image.
|
||||
pub(crate) fn write_v7(frozen: &FrozenTrie, meta: &TrieMetadata) -> io::Result<Vec<u8>> {
|
||||
let stream = write_stream(frozen, meta)?;
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(&stream)?;
|
||||
encoder.finish()
|
||||
}
|
||||
|
||||
// Compiled-stream reader.
|
||||
|
||||
struct Reader<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
fn new(data: &'a [u8]) -> Self {
|
||||
Reader { data, pos: 0 }
|
||||
}
|
||||
|
||||
fn take(&mut self, n: usize) -> io::Result<&'a [u8]> {
|
||||
if self.pos + n > self.data.len() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"unexpected end of trie stream",
|
||||
));
|
||||
}
|
||||
let slice = &self.data[self.pos..self.pos + n];
|
||||
self.pos += n;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
fn i32(&mut self) -> io::Result<i32> {
|
||||
let b = self.take(4)?;
|
||||
Ok(i32::from_be_bytes([b[0], b[1], b[2], b[3]]))
|
||||
}
|
||||
|
||||
fn u16(&mut self) -> io::Result<u16> {
|
||||
let b = self.take(2)?;
|
||||
Ok(u16::from_be_bytes([b[0], b[1]]))
|
||||
}
|
||||
|
||||
fn u8(&mut self) -> io::Result<u8> {
|
||||
Ok(self.take(1)?[0])
|
||||
}
|
||||
|
||||
fn java_utf(&mut self) -> io::Result<String> {
|
||||
let len = self.u16()? as usize;
|
||||
let bytes = self.take(len)?;
|
||||
decode_java_utf(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_java_utf(bytes: &[u8]) -> io::Result<String> {
|
||||
let mut units: Vec<u16> = Vec::with_capacity(bytes.len());
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
let b = bytes[i];
|
||||
if b & 0x80 == 0 {
|
||||
units.push(b as u16);
|
||||
i += 1;
|
||||
} else if b & 0xE0 == 0xC0 {
|
||||
if i + 1 >= bytes.len() {
|
||||
return Err(malformed());
|
||||
}
|
||||
let b1 = bytes[i + 1];
|
||||
units.push((((b as u16 & 0x1F) << 6) | (b1 as u16 & 0x3F)) as u16);
|
||||
i += 2;
|
||||
} else if b & 0xF0 == 0xE0 {
|
||||
if i + 2 >= bytes.len() {
|
||||
return Err(malformed());
|
||||
}
|
||||
let b1 = bytes[i + 1];
|
||||
let b2 = bytes[i + 2];
|
||||
units.push(((b as u16 & 0x0F) << 12) | ((b1 as u16 & 0x3F) << 6) | (b2 as u16 & 0x3F));
|
||||
i += 3;
|
||||
} else {
|
||||
return Err(malformed());
|
||||
}
|
||||
}
|
||||
Ok(String::from_utf16_lossy(&units))
|
||||
}
|
||||
|
||||
fn malformed() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "malformed modified UTF-8")
|
||||
}
|
||||
|
||||
fn parse_metadata(text: &str) -> TrieMetadata {
|
||||
let mut traversal = TraversalDirection::Backward;
|
||||
let mut case_mode = CaseMode::LowercaseWithLocaleRoot;
|
||||
let mut diacritic_mode = DiacriticMode::AsIs;
|
||||
for line in text.lines() {
|
||||
if let Some((key, value)) = line.split_once('=') {
|
||||
match key {
|
||||
"traversalDirection" => {
|
||||
traversal = if value == "FORWARD" {
|
||||
TraversalDirection::Forward
|
||||
} else {
|
||||
TraversalDirection::Backward
|
||||
};
|
||||
}
|
||||
"caseProcessingMode" => {
|
||||
case_mode = if value == "AS_IS" {
|
||||
CaseMode::AsIs
|
||||
} else {
|
||||
CaseMode::LowercaseWithLocaleRoot
|
||||
};
|
||||
}
|
||||
"diacriticProcessingMode" => {
|
||||
diacritic_mode = if value == "REMOVE" {
|
||||
DiacriticMode::Remove
|
||||
} else {
|
||||
DiacriticMode::AsIs
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
TrieMetadata {
|
||||
traversal,
|
||||
case_mode,
|
||||
diacritic_mode,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild dense direct-index tables from the CSR edges (same policy as freeze).
|
||||
fn build_dense(
|
||||
edge_start: &[u32],
|
||||
edge_labels: &[u16],
|
||||
edge_targets: &[u32],
|
||||
) -> (Vec<u32>, Vec<u16>, Vec<u32>) {
|
||||
let node_count = edge_start.len() - 1;
|
||||
let mut dense_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut dense_base: Vec<u16> = Vec::with_capacity(node_count);
|
||||
let mut dense_targets: Vec<u32> = Vec::new();
|
||||
dense_start.push(0);
|
||||
for node in 0..node_count {
|
||||
let lo = edge_start[node] as usize;
|
||||
let hi = edge_start[node + 1] as usize;
|
||||
let count = hi - lo;
|
||||
let mut dense = false;
|
||||
if count >= 2 {
|
||||
let first = edge_labels[lo] as usize;
|
||||
let last = edge_labels[hi - 1] as usize;
|
||||
let span = last - first + 1;
|
||||
if span <= MAX_DENSE_SPAN {
|
||||
let base = edge_labels[lo];
|
||||
let seg = dense_targets.len();
|
||||
dense_targets.resize(seg + span, 0);
|
||||
for k in lo..hi {
|
||||
dense_targets[seg + (edge_labels[k] - base) as usize] = edge_targets[k] + 1;
|
||||
}
|
||||
dense_base.push(base);
|
||||
dense_start.push(dense_targets.len() as u32);
|
||||
dense = true;
|
||||
}
|
||||
}
|
||||
if !dense {
|
||||
dense_base.push(0);
|
||||
dense_start.push(dense_targets.len() as u32);
|
||||
}
|
||||
}
|
||||
(dense_start, dense_base, dense_targets)
|
||||
}
|
||||
|
||||
/// Read a gzip-compressed Java v7 compiled-trie image into a runtime trie.
|
||||
#[allow(dead_code)] // convenience wrapper; lib.rs decompresses then calls read_stream
|
||||
pub(crate) fn read_v7(gz_bytes: &[u8]) -> io::Result<FrequencyTrie> {
|
||||
let mut data = Vec::new();
|
||||
GzDecoder::new(gz_bytes).read_to_end(&mut data)?;
|
||||
read_stream(&data)
|
||||
}
|
||||
|
||||
/// Whether `decompressed` (an already-gunzipped byte stream) is a v7 trie image.
|
||||
pub(crate) fn is_v7_stream(decompressed: &[u8]) -> bool {
|
||||
decompressed.len() >= 4
|
||||
&& i32::from_be_bytes([
|
||||
decompressed[0],
|
||||
decompressed[1],
|
||||
decompressed[2],
|
||||
decompressed[3],
|
||||
]) == STREAM_MAGIC
|
||||
}
|
||||
|
||||
/// Parse the inner (uncompressed) v7 stream into a runtime trie.
|
||||
pub(crate) fn read_stream(data: &[u8]) -> io::Result<FrequencyTrie> {
|
||||
let mut r = Reader::new(data);
|
||||
if r.i32()? != STREAM_MAGIC {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"bad trie stream magic",
|
||||
));
|
||||
}
|
||||
let version = r.i32()?;
|
||||
if version != STREAM_VERSION {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("unsupported trie stream version {version} (expected {STREAM_VERSION})"),
|
||||
));
|
||||
}
|
||||
let node_count = r.i32()? as usize;
|
||||
let root_id = r.i32()?;
|
||||
if root_id != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unsupported non-zero root node id",
|
||||
));
|
||||
}
|
||||
let metadata = parse_metadata(&r.java_utf()?);
|
||||
let backward = matches!(metadata.traversal, TraversalDirection::Backward);
|
||||
|
||||
let value_table_len = r.i32()? as usize;
|
||||
let mut value_table: Vec<Arc<PatchCommand>> = Vec::with_capacity(value_table_len);
|
||||
for _ in 0..value_table_len {
|
||||
let patch = r.java_utf()?;
|
||||
value_table.push(Arc::new(PatchCommand::parse(&patch, backward)));
|
||||
}
|
||||
|
||||
let mut edge_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut edge_labels: Vec<u16> = Vec::new();
|
||||
let mut edge_targets: Vec<u32> = Vec::new();
|
||||
let mut accepts: Vec<bool> = Vec::with_capacity(node_count);
|
||||
let mut value_start: Vec<u32> = Vec::with_capacity(node_count + 1);
|
||||
let mut values: Vec<Arc<PatchCommand>> = Vec::new();
|
||||
edge_start.push(0);
|
||||
value_start.push(0);
|
||||
|
||||
for _ in 0..node_count {
|
||||
accepts.push(r.u8()? != 0);
|
||||
let edge_count = r.i32()? as usize;
|
||||
for _ in 0..edge_count {
|
||||
let label = r.u16()?;
|
||||
let child = r.i32()? as u32;
|
||||
edge_labels.push(label);
|
||||
edge_targets.push(child);
|
||||
}
|
||||
edge_start.push(edge_labels.len() as u32);
|
||||
|
||||
let value_count = r.i32()? as usize;
|
||||
for _ in 0..value_count {
|
||||
let value_id = r.i32()? as usize;
|
||||
let _count = r.i32()?; // frequency: not used at runtime
|
||||
if value_id >= value_table.len() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"value id out of range",
|
||||
));
|
||||
}
|
||||
values.push(Arc::clone(&value_table[value_id]));
|
||||
}
|
||||
value_start.push(values.len() as u32);
|
||||
}
|
||||
|
||||
let (dense_start, dense_base, dense_targets) =
|
||||
build_dense(&edge_start, &edge_labels, &edge_targets);
|
||||
|
||||
Ok(FrequencyTrie::new(
|
||||
edge_start,
|
||||
edge_labels,
|
||||
edge_targets,
|
||||
accepts,
|
||||
value_start,
|
||||
values,
|
||||
dense_start,
|
||||
dense_base,
|
||||
dense_targets,
|
||||
metadata,
|
||||
))
|
||||
}
|
||||
294
python/src/trie.rs
Normal file
294
python/src/trie.rs
Normal file
@@ -0,0 +1,294 @@
|
||||
// Copyright (C) 2026, Leo Galambos
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
// may be used to endorse or promote products derived from this software
|
||||
// without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
use crate::patch::PatchCommand;
|
||||
use std::borrow::Cow;
|
||||
use std::sync::Arc;
|
||||
use unicode_general_category::{get_general_category, GeneralCategory};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TraversalDirection {
|
||||
Backward,
|
||||
Forward,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CaseMode {
|
||||
LowercaseWithLocaleRoot,
|
||||
AsIs,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DiacriticMode {
|
||||
AsIs,
|
||||
Remove,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TrieMetadata {
|
||||
pub traversal: TraversalDirection,
|
||||
pub case_mode: CaseMode,
|
||||
pub diacritic_mode: DiacriticMode,
|
||||
}
|
||||
|
||||
/// Compiled patch-command trie in a flat, cache-friendly CSR layout.
|
||||
///
|
||||
/// Instead of a graph of heap-allocated, reference-counted nodes (which forces
|
||||
/// a pointer chase and a likely cache miss at every character step), the whole
|
||||
/// trie is stored as a handful of contiguous arrays indexed by node id:
|
||||
///
|
||||
/// * `edge_start[i] .. edge_start[i+1]` slices `edge_labels` / `edge_targets`
|
||||
/// for node `i` (labels sorted ascending, so child lookup is a binary search
|
||||
/// over a contiguous, cache-hot slice — no pointer chasing, no atomics),
|
||||
/// * `accepts[i]` marks a contracted accepting leaf,
|
||||
/// * `value_start[i] .. value_start[i+1]` slices `values` (best value first).
|
||||
///
|
||||
/// Node 0 is the root. Shared (deduplicated) subtrees simply reference the same
|
||||
/// node id, so structural sharing from reduction is preserved without `Arc`.
|
||||
pub struct FrequencyTrie {
|
||||
edge_start: Vec<u32>,
|
||||
edge_labels: Vec<u16>,
|
||||
edge_targets: Vec<u32>,
|
||||
accepts: Vec<bool>,
|
||||
value_start: Vec<u32>,
|
||||
values: Vec<Arc<PatchCommand>>,
|
||||
// Adaptive child lookup (mirrors the Java CompiledNode fanout strategy):
|
||||
// high-fanout nodes whose child labels span a small contiguous range get a
|
||||
// dense direct-index table (O(1) child access); sparse nodes fall back to
|
||||
// binary search over `edge_labels`. A node `i` is dense iff
|
||||
// `dense_start[i+1] > dense_start[i]`; then `dense_targets[dense_start[i] +
|
||||
// (label - dense_base[i])]` holds `child_id + 1` (0 = no such edge).
|
||||
dense_start: Vec<u32>,
|
||||
dense_base: Vec<u16>,
|
||||
dense_targets: Vec<u32>,
|
||||
pub metadata: TrieMetadata,
|
||||
}
|
||||
|
||||
impl FrequencyTrie {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
edge_start: Vec<u32>,
|
||||
edge_labels: Vec<u16>,
|
||||
edge_targets: Vec<u32>,
|
||||
accepts: Vec<bool>,
|
||||
value_start: Vec<u32>,
|
||||
values: Vec<Arc<PatchCommand>>,
|
||||
dense_start: Vec<u32>,
|
||||
dense_base: Vec<u16>,
|
||||
dense_targets: Vec<u32>,
|
||||
metadata: TrieMetadata,
|
||||
) -> Self {
|
||||
FrequencyTrie {
|
||||
edge_start,
|
||||
edge_labels,
|
||||
edge_targets,
|
||||
accepts,
|
||||
value_start,
|
||||
values,
|
||||
dense_start,
|
||||
dense_base,
|
||||
dense_targets,
|
||||
metadata,
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalize a lookup key (used by the rare diacritic-removal path and by
|
||||
/// `stem_all`). Borrows the input when no transformation is needed.
|
||||
fn normalize_key<'a>(&self, word: &'a str) -> Cow<'a, str> {
|
||||
let lowered: Cow<'a, str> =
|
||||
if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot)
|
||||
&& word.chars().any(|c| c.is_uppercase())
|
||||
{
|
||||
Cow::Owned(word.to_lowercase())
|
||||
} else {
|
||||
Cow::Borrowed(word)
|
||||
};
|
||||
if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) {
|
||||
Cow::Owned(strip_diacritics(&lowered))
|
||||
} else {
|
||||
lowered
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode the normalized lookup key into `key_buf` in a single pass over the
|
||||
/// input: lowercasing (when configured) is folded into the UTF-16 encoding
|
||||
/// so the UTF-8 input is decoded only once and no intermediate `String` is
|
||||
/// allocated. The diacritic-removal path (unused by the bundled models)
|
||||
/// falls back to the general `normalize_key`.
|
||||
#[inline]
|
||||
fn encode_key(&self, word: &str, key_buf: &mut Vec<u16>) {
|
||||
key_buf.clear();
|
||||
if matches!(self.metadata.diacritic_mode, DiacriticMode::Remove) {
|
||||
let normalized = self.normalize_key(word);
|
||||
key_buf.extend(normalized.encode_utf16());
|
||||
return;
|
||||
}
|
||||
if matches!(self.metadata.case_mode, CaseMode::LowercaseWithLocaleRoot) {
|
||||
let mut unit = [0u16; 2];
|
||||
for c in word.chars() {
|
||||
if c.is_ascii() {
|
||||
// ASCII fast path: lowercasing requires a single branch.
|
||||
key_buf.push(c.to_ascii_lowercase() as u16);
|
||||
} else if c.is_lowercase() {
|
||||
// Already lowercase (e.g. lowercase Cyrillic/Greek): encode
|
||||
// directly and skip the costly Unicode special-casing.
|
||||
key_buf.extend_from_slice(c.encode_utf16(&mut unit));
|
||||
} else {
|
||||
for lc in c.to_lowercase() {
|
||||
key_buf.extend_from_slice(lc.encode_utf16(&mut unit));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
key_buf.extend(word.encode_utf16());
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the child of `node` on `label` via binary search over the node's
|
||||
/// contiguous, ascending edge-label slice. Uses unchecked indexing on
|
||||
/// provably in-range offsets to drop bounds checks from the hot loop.
|
||||
#[inline]
|
||||
fn child(&self, node: usize, label: u16) -> Option<usize> {
|
||||
// Dense high-fanout node: O(1) direct index.
|
||||
// SAFETY: node and node+1 index dense_start (len = num_nodes+1).
|
||||
let ds = unsafe { *self.dense_start.get_unchecked(node) } as usize;
|
||||
let de = unsafe { *self.dense_start.get_unchecked(node + 1) } as usize;
|
||||
if de > ds {
|
||||
let base = unsafe { *self.dense_base.get_unchecked(node) };
|
||||
let idx = label.wrapping_sub(base) as usize;
|
||||
if idx < de - ds {
|
||||
// SAFETY: ds + idx < de <= dense_targets.len().
|
||||
let t = unsafe { *self.dense_targets.get_unchecked(ds + idx) };
|
||||
if t != 0 {
|
||||
return Some((t - 1) as usize);
|
||||
}
|
||||
}
|
||||
return None;
|
||||
}
|
||||
// Sparse node: binary search over the contiguous ascending edge slice.
|
||||
// SAFETY: node and node+1 index edge_start (len = num_nodes+1).
|
||||
let lo = unsafe { *self.edge_start.get_unchecked(node) } as usize;
|
||||
let hi = unsafe { *self.edge_start.get_unchecked(node + 1) } as usize;
|
||||
// SAFETY: lo <= hi <= edge_labels.len() by construction.
|
||||
let labels = unsafe { self.edge_labels.get_unchecked(lo..hi) };
|
||||
match labels.binary_search(&label) {
|
||||
// SAFETY: lo+pos < hi <= edge_targets.len().
|
||||
Ok(pos) => Some(unsafe { *self.edge_targets.get_unchecked(lo + pos) } as usize),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the trie for `key`, returning the accepting/terminal node id.
|
||||
#[inline]
|
||||
fn find_node(&self, key: &[u16]) -> Option<usize> {
|
||||
let mut node = 0usize;
|
||||
match self.metadata.traversal {
|
||||
TraversalDirection::Backward => {
|
||||
for &label in key.iter().rev() {
|
||||
if unsafe { *self.accepts.get_unchecked(node) } {
|
||||
return Some(node);
|
||||
}
|
||||
node = self.child(node, label)?;
|
||||
}
|
||||
}
|
||||
TraversalDirection::Forward => {
|
||||
for &label in key.iter() {
|
||||
if unsafe { *self.accepts.get_unchecked(node) } {
|
||||
return Some(node);
|
||||
}
|
||||
node = self.child(node, label)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(node)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn preferred_value(&self, node: usize) -> Option<&Arc<PatchCommand>> {
|
||||
let start = self.value_start[node] as usize;
|
||||
let end = self.value_start[node + 1] as usize;
|
||||
if start == end {
|
||||
None
|
||||
} else {
|
||||
Some(&self.values[start])
|
||||
}
|
||||
}
|
||||
|
||||
/// Stem into caller-owned scratch buffers and return the produced length
|
||||
/// without allocating an output String. This also supports diagnostics
|
||||
/// that isolate the algorithm from output-String allocation.
|
||||
pub fn stem_len_into(
|
||||
&self,
|
||||
word: &str,
|
||||
key_buf: &mut Vec<u16>,
|
||||
out_buf: &mut Vec<u16>,
|
||||
) -> Option<usize> {
|
||||
self.encode_key(word, key_buf);
|
||||
let node = self.find_node(key_buf)?;
|
||||
let patch = self.preferred_value(node)?;
|
||||
patch.apply_into(key_buf, out_buf);
|
||||
Some(out_buf.len())
|
||||
}
|
||||
|
||||
/// Diagnostic: only normalize + UTF-16 encode the key.
|
||||
pub fn bench_encode(&self, word: &str, key_buf: &mut Vec<u16>) -> usize {
|
||||
self.encode_key(word, key_buf);
|
||||
key_buf.len()
|
||||
}
|
||||
|
||||
/// Diagnostic: normalize + encode + trie walk (no patch apply).
|
||||
pub fn bench_find(&self, word: &str, key_buf: &mut Vec<u16>) -> bool {
|
||||
self.encode_key(word, key_buf);
|
||||
self.find_node(key_buf).is_some()
|
||||
}
|
||||
|
||||
/// Return all stems in frequency order.
|
||||
pub fn stem_all(&self, word: &str) -> Vec<String> {
|
||||
let mut key_u16: Vec<u16> = Vec::new();
|
||||
self.encode_key(word, &mut key_u16);
|
||||
match self.find_node(&key_u16) {
|
||||
None => Vec::new(),
|
||||
Some(node) => {
|
||||
let start = self.value_start[node] as usize;
|
||||
let end = self.value_start[node + 1] as usize;
|
||||
self.values[start..end]
|
||||
.iter()
|
||||
.map(|p| String::from_utf16_lossy(&p.apply(&key_u16)))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn strip_diacritics(s: &str) -> String {
|
||||
s.nfd()
|
||||
.filter(|ch| !matches!(get_general_category(*ch), GeneralCategory::NonspacingMark))
|
||||
.collect()
|
||||
}
|
||||
Reference in New Issue
Block a user