Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: EGOTHOR v4 hot-path additions

Browse Source
This commit is contained in:
Leo Galambos
2026-05-17 15:00:45 +02:00
parent 7bd0fc66ba
commit 87ff85fd6d
16 changed files with 1877 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
src/jmh/java/org/egothor/stemmer/benchmark/FrequencyTrieLookupBenchmark.java
View File

@@ -31,11 +31,13 @@
package org.egothor.stemmer.benchmark;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.PatchCommandEncoder;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.ReductionSettings;
import org.egothor.stemmer.ValueCount;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
@@ -97,12 +99,45 @@ public class FrequencyTrieLookupBenchmark {
*/
private String[] lookupKeys;
/**
* Lookup keys as normalized caller-owned character storage.
*/
private char[][] lookupKeyCharacters;
/**
* Keys that are known to return multiple patch candidates from
* {@code getAll()}.
*/
private String[] ambiguousLookupKeys;
/**
* Ambiguous lookup keys as normalized caller-owned character storage.
*/
private char[][] ambiguousLookupKeyCharacters;
/**
* Preferred patches aligned with {@link #lookupKeys}.
*/
private String[] preferredPatches;
/**
* Reusable output buffer for patch application benchmarks.
*/
private char[] outputBuffer;
/**
* Mutable field consumed by visitor sinks.
*/
private int visitorAccumulator;
/**
* Sink used by visitor lookup benchmarks without per-invocation allocation.
*/
private final FrequencyTrie.EntrySink<String> visitorSink = (value, count, rank) -> {
this.visitorAccumulator += value.length() + count + rank;
return true;
};
/**
* Initializes the benchmark state.
*
@@ -116,6 +151,23 @@ public class FrequencyTrieLookupBenchmark {
this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true);
this.lookupKeys = corpus.lookupKeys();
this.ambiguousLookupKeys = corpus.ambiguousLookupKeys();
this.lookupKeyCharacters = toCharArrays(this.lookupKeys);
this.ambiguousLookupKeyCharacters = toCharArrays(this.ambiguousLookupKeys);
this.preferredPatches = new String[this.lookupKeys.length];
int maxKeyLength = 0;
for (int index = 0; index < this.lookupKeys.length; index++) {
this.preferredPatches[index] = this.trie.get(this.lookupKeys[index]);
maxKeyLength = Math.max(maxKeyLength, this.lookupKeys[index].length());
}
this.outputBuffer = new char[maxKeyLength + 32];
}
private static char[][] toCharArrays(final String[] values) {
final char[][] characters = new char[values.length][];
for (int index = 0; index < values.length; index++) {
characters[index] = values[index].toCharArray();
}
return characters;
}
}
@@ -155,6 +207,61 @@ public class FrequencyTrieLookupBenchmark {
}
}
/**
* Measures retrieval of all patch candidates through caller-owned normalized
* character storage and a visitor sink.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupAllPatchesWithNormalizedCharVisitor(final LookupState state, final Blackhole blackhole) {
final char[][] keys = state.ambiguousLookupKeyCharacters;
for (char[] key : keys) {
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
if (count < 2) {
throw new IllegalStateException("Expected multiple patches for benchmark key.");
}
}
blackhole.consume(state.visitorAccumulator);
}
/**
* Measures counted candidate retrieval through the allocating entry API.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupPatchEntries(final LookupState state, final Blackhole blackhole) {
final String[] keys = state.ambiguousLookupKeys;
for (String key : keys) {
final List<ValueCount<String>> entries = state.trie.getEntries(key);
if (entries.size() < 2) {
throw new IllegalStateException("Expected multiple entries for key " + key + '.');
}
blackhole.consume(entries);
}
}
/**
* Measures counted candidate retrieval through the visitor API.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupPatchEntriesWithVisitor(final LookupState state, final Blackhole blackhole) {
final char[][] keys = state.ambiguousLookupKeyCharacters;
for (char[] key : keys) {
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
if (count < 2) {
throw new IllegalStateException("Expected multiple entries for benchmark key.");
}
}
blackhole.consume(state.visitorAccumulator);
}
/**
* Measures end-to-end preferred stemming from lookup plus patch application.
*
@@ -170,6 +277,28 @@ public class FrequencyTrieLookupBenchmark {
}
}
/**
* Measures patch application into caller-owned output storage.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void applyPreferredPatchToBuffer(final LookupState state, final Blackhole blackhole) {
final String[] keys = state.lookupKeys;
final String[] patches = state.preferredPatches;
final char[] output = state.outputBuffer;
for (int index = 0; index < keys.length; index++) {
final int length = PatchCommandEncoder.applyTo(keys[index], patches[index],
state.trie.traversalDirection(), output, 0, output.length);
if (length == PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
throw new IllegalStateException("Output buffer too small for key " + keys[index] + '.');
}
blackhole.consume(length);
blackhole.consume(output[0]);
}
}
/**
* Measures end-to-end full candidate stemming from {@code getAll()} plus
* patch application.

3
src/main/java/org/egothor/stemmer/CaseProcessingMode.java
View File

@@ -48,8 +48,7 @@ public enum CaseProcessingMode {
AS_IS,
/**
* Normalizes all dictionary content to lower case using
* {@link Locale#ROOT}.
* Normalizes all dictionary content to lower case using {@link Locale#ROOT}.
*/
LOWERCASE_WITH_LOCALE_ROOT
}

8
src/main/java/org/egothor/stemmer/DiacriticStripper.java
View File

@@ -93,12 +93,12 @@ final class DiacriticStripper {
}
/**
* Removes supported diacritic marks and common Latin ligatures from the supplied
* text.
* Removes supported diacritic marks and common Latin ligatures from the
* supplied text.
*
* <p>
* The method returns the original {@link String} instance when no replacement is
* required, avoiding an unnecessary allocation on the common ASCII path.
* The method returns the original {@link String} instance when no replacement
* is required, avoiding an unnecessary allocation on the common ASCII path.
* </p>
*
* @param input text to normalize

344
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -119,7 +119,8 @@ public final class FrequencyTrie<V> {
private final boolean removeDiacritics;
/**
* Shared empty array instance for empty lookup results from {@link #getAll(String)}.
* Shared empty array instance for empty lookup results from
* {@link #getAll(String)}.
*/
private final V[] emptyValues;
@@ -165,13 +166,18 @@ public final class FrequencyTrie<V> {
*/
private static final int CASE_VERSION = 4;
/**
* Argument name for lookup keys.
*/
private static final String ARG_KEY = "key";
/**
* Default dense child lookup span in code points used when materializing
* compiled nodes without an explicit override.
* <p>
* Increasing this value increases the chance of direct array indexing for
* child lookup at runtime at the cost of per-node dense table memory for
* compact character spans.
* Increasing this value increases the chance of direct array indexing for child
* lookup at runtime at the cost of per-node dense table memory for compact
* character spans.
* </p>
*/
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
@@ -191,6 +197,30 @@ public final class FrequencyTrie<V> {
return STREAM_VERSION;
}
/**
* Receives trie values during visitor-style lookup.
*
* <p>
* Implementations are caller-owned and are not retained by the trie. Returning
* {@code false} stops iteration after the current callback.
* </p>
*
* @param <V> value type
*/
@FunctionalInterface
public interface EntrySink<V> {
/**
* Accepts one ordered local value.
*
* @param value stored value
* @param count stored local occurrence count
* @param rank zero-based rank in deterministic local ordering
* @return {@code true} to continue iteration, {@code false} to stop
*/
boolean accept(V value, int count, int rank);
}
/**
* Creates a new compiled trie instance.
*
@@ -229,7 +259,7 @@ public final class FrequencyTrie<V> {
* @throws NullPointerException if {@code key} is {@code null}
*/
public V get(final String key) {
Objects.requireNonNull(key, "key");
Objects.requireNonNull(key, ARG_KEY);
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
if (node == null) {
return null;
@@ -266,7 +296,7 @@ public final class FrequencyTrie<V> {
*/
@SuppressWarnings("PMD.MethodReturnsInternalArray")
public V[] getAll(final String key) {
Objects.requireNonNull(key, "key");
Objects.requireNonNull(key, ARG_KEY);
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
if (node == null) {
return this.emptyValues;
@@ -301,7 +331,7 @@ public final class FrequencyTrie<V> {
* @throws NullPointerException if {@code key} is {@code null}
*/
public List<ValueCount<V>> getEntries(final String key) {
Objects.requireNonNull(key, "key");
Objects.requireNonNull(key, ARG_KEY);
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
if (node == null) {
return List.of();
@@ -325,6 +355,132 @@ public final class FrequencyTrie<V> {
return Collections.unmodifiableList(entries);
}
/**
* Visits all values stored at the node addressed by an already-normalized
* {@code char[]} key slice.
*
* <p>
* This method bypasses {@link TrieMetadata#caseProcessingMode()} and
* {@link TrieMetadata#diacriticProcessingMode()}. The caller must provide input
* normalized exactly as required by this trie's metadata. The trie is immutable
* and thread-safe for concurrent reads; the supplied sink is caller-owned and
* is not retained.
* </p>
*
* @param key normalized key storage
* @param offset first character offset
* @param length number of characters to read
* @param sink value sink
* @param maxResults maximum number of results to visit
* @return number of visited values
* @throws NullPointerException if {@code key} or {@code sink} is
* {@code null}
* @throws IndexOutOfBoundsException if the key slice is invalid
* @throws IllegalArgumentException if {@code maxResults} is negative
*/
public int getAllNormalized(final char[] key, final int offset, final int length, final EntrySink<? super V> sink,
final int maxResults) {
Objects.requireNonNull(key, ARG_KEY);
Objects.requireNonNull(sink, "sink");
Objects.checkFromIndexSize(offset, length, key.length);
validateMaxResults(maxResults);
if (maxResults == 0) {
return 0;
}
return visitNode(findNode(key, offset, length), sink, maxResults);
}
/**
* Visits all values stored at the node addressed by an already-normalized
* character sequence.
*
* @param key normalized key
* @param sink value sink
* @param maxResults maximum number of results to visit
* @return number of visited values
* @throws NullPointerException if {@code key} or {@code sink} is
* {@code null}
* @throws IllegalArgumentException if {@code maxResults} is negative
* @see #getAllNormalized(char[], int, int, EntrySink, int)
*/
public int getAllNormalized(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
Objects.requireNonNull(key, ARG_KEY);
Objects.requireNonNull(sink, "sink");
validateMaxResults(maxResults);
if (maxResults == 0) {
return 0;
}
return visitNode(findNode(key), sink, maxResults);
}
/**
* Visits the first value stored at the node addressed by an already-normalized
* {@code char[]} key slice.
*
* @param key normalized key storage
* @param offset first character offset
* @param length number of characters to read
* @param sink value sink
* @return {@code true} when a value was visited, otherwise {@code false}
* @see #getAllNormalized(char[], int, int, EntrySink, int)
*/
public boolean getFirstNormalized(final char[] key, final int offset, final int length,
final EntrySink<? super V> sink) {
return getAllNormalized(key, offset, length, sink, 1) == 1;
}
/**
* Visits the first value stored at the node addressed by an already-normalized
* character sequence.
*
* @param key normalized key
* @param sink value sink
* @return {@code true} when a value was visited, otherwise {@code false}
* @see #getAllNormalized(CharSequence, EntrySink, int)
*/
public boolean getFirstNormalized(final CharSequence key, final EntrySink<? super V> sink) {
return getAllNormalized(key, sink, 1) == 1;
}
/**
* Visits all values stored at the node addressed by the supplied key, applying
* metadata-driven lookup normalization when required.
*
* <p>
* This method preserves the same lookup normalization semantics as
* {@link #getAll(String)}. It may allocate when metadata requires lowercase or
* diacritic normalization.
* </p>
*
* @param key key to resolve
* @param sink value sink
* @param maxResults maximum number of results to visit
* @return number of visited values
*/
public int getAll(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
Objects.requireNonNull(key, ARG_KEY);
Objects.requireNonNull(sink, "sink");
validateMaxResults(maxResults);
if (maxResults == 0) {
return 0;
}
final CharSequence normalized = normalizeLookupKey(key);
return visitNode(findNode(normalized), sink, maxResults);
}
/**
* Visits the first value stored at the node addressed by the supplied key,
* applying metadata-driven lookup normalization when required.
*
* @param key key to resolve
* @param sink value sink
* @return {@code true} when a value was visited, otherwise {@code false}
* @see #getAll(CharSequence, EntrySink, int)
*/
public boolean getFirst(final CharSequence key, final EntrySink<? super V> sink) {
return getAll(key, sink, 1) == 1;
}
/**
* Returns the logical key traversal direction used by this trie.
*
@@ -431,16 +587,17 @@ public final class FrequencyTrie<V> {
* dense child-index span configuration.
* <p>
* This setting is applied only while materializing the in-memory compiled
* representation during load. It is not serialized in {@link TrieMetadata},
* so each load can independently choose its own runtime lookup trade-off.
* representation during load. It is not serialized in {@link TrieMetadata}, so
* each load can independently choose its own runtime lookup trade-off.
* </p>
*
* @param inputStream source input stream
* @param arrayFactory array factory used to create typed arrays
* @param valueCodec codec used to read values
* @param maxExpandedIndex dense lookup span override; zero disables dense lookup,
* negative values use {@link #DEFAULT_MAX_EXPANDED_INDEX}
* @param <V> value type
* @param inputStream source input stream
* @param arrayFactory array factory used to create typed arrays
* @param valueCodec codec used to read values
* @param maxExpandedIndex dense lookup span override; zero disables dense
* lookup, negative values use
* {@link #DEFAULT_MAX_EXPANDED_INDEX}
* @param <V> value type
* @return deserialized compiled trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if reading fails or the binary format is invalid
@@ -573,7 +730,8 @@ public final class FrequencyTrie<V> {
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount, effectiveMaxExpandedIndex);
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount,
effectiveMaxExpandedIndex);
final CompiledNode<V> rootNode = nodes[rootNodeId];
if (LOGGER.isLoggable(Level.FINE)) {
@@ -584,12 +742,12 @@ public final class FrequencyTrie<V> {
}
private static DataInputStream wrapInputStream(final InputStream inputStream) {
return inputStream instanceof DataInputStream
? (DataInputStream) inputStream
return inputStream instanceof DataInputStream ? (DataInputStream) inputStream
: new DataInputStream(inputStream);
}
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version) throws IOException {
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version)
throws IOException {
if (version == STREAM_VERSION) {
return readTextMetadata(dataInput);
}
@@ -600,12 +758,12 @@ public final class FrequencyTrie<V> {
}
final ReductionSettings reductionSettings = readReductionSettings(dataInput);
final DiacriticProcessingMode diacriticProcessingMode = readEnumByOrdinal(dataInput, DiacriticProcessingMode.values(),
"diacritic processing mode");
final CaseProcessingMode caseProcessingMode = version >= CASE_VERSION
? readCaseProcessingMode(dataInput)
final DiacriticProcessingMode diacriticProcessingMode = readEnumByOrdinal(dataInput,
DiacriticProcessingMode.values(), "diacritic processing mode");
final CaseProcessingMode caseProcessingMode = version >= CASE_VERSION ? readCaseProcessingMode(dataInput)
: CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
return new TrieMetadata(version, traversalDirection, reductionSettings, diacriticProcessingMode, caseProcessingMode);
return new TrieMetadata(version, traversalDirection, reductionSettings, diacriticProcessingMode,
caseProcessingMode);
}
private static TrieMetadata readTextMetadata(final DataInputStream dataInput) throws IOException {
@@ -644,8 +802,9 @@ public final class FrequencyTrie<V> {
return values[ordinal];
}
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
final ValueStreamCodec<V> valueCodec, final int nodeCount, final int maxExpandedIndex) throws IOException {
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final int nodeCount,
final int maxExpandedIndex) throws IOException {
final char[][] edgeLabelsByNode = new char[nodeCount][];
final int[][] childNodeIdsByNode = new int[nodeCount][];
@SuppressWarnings("unchecked")
@@ -700,14 +859,16 @@ public final class FrequencyTrie<V> {
private static <V> CompiledNode<V> resolveNode(final int nodeIndex, final char[][] edgeLabelsByNode,
final int[][] childNodeIdsByNode, final V[][] orderedValuesByNode, final int[][] orderedCountsByNode,
final CompiledNode<V>[] nodes, final boolean[] inProgress, final int maxExpandedIndex) throws IOException {
final CompiledNode<V>[] nodes, final boolean[] inProgress, final int maxExpandedIndex)
throws IOException {
final CompiledNode<V> cachedNode = nodes[nodeIndex];
if (cachedNode != null) {
return cachedNode;
}
if (inProgress[nodeIndex]) {
throw new IOException("Invalid serialized node graph: cyclic reference detected at node " + nodeIndex + '.');
throw new IOException(
"Invalid serialized node graph: cyclic reference detected at node " + nodeIndex + '.');
}
inProgress[nodeIndex] = true;
try {
@@ -720,16 +881,15 @@ public final class FrequencyTrie<V> {
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
final int childNodeId = childNodeIds[edgeIndex];
if (childNodeId < 0 || childNodeId >= edgeLabelsByNode.length) {
throw new IOException(
"Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex + ": "
+ childNodeId);
throw new IOException("Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex
+ ": " + childNodeId);
}
children[edgeIndex] = resolveNode(childNodeId, edgeLabelsByNode, childNodeIdsByNode,
orderedValuesByNode, orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
}
final CompiledNode<V> node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex], maxExpandedIndex,
orderedCountsByNode[nodeIndex]);
final CompiledNode<V> node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex],
maxExpandedIndex, orderedCountsByNode[nodeIndex]);
nodes[nodeIndex] = node;
return node;
} finally {
@@ -740,8 +900,9 @@ public final class FrequencyTrie<V> {
private static void validateSerializedEdges(final int nodeIndex, final char... edgeLabels) throws IOException {
for (int edgeIndex = 1; edgeIndex < edgeLabels.length; edgeIndex++) {
if (edgeLabels[edgeIndex - 1] >= edgeLabels[edgeIndex]) {
throw new IOException("Edge labels must be strictly ascending at node " + nodeIndex + ", edge index "
+ edgeIndex + ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
throw new IOException(
"Edge labels must be strictly ascending at node " + nodeIndex + ", edge index " + edgeIndex
+ ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
}
}
}
@@ -754,6 +915,16 @@ public final class FrequencyTrie<V> {
* @return compiled node, or {@code null} if the path does not exist
*/
private CompiledNode<V> findNode(final String key) {
return findNode((CharSequence) key);
}
/**
* Locates the compiled node for the supplied key.
*
* @param key already-normalized key to resolve
* @return compiled node, or {@code null} if the path does not exist
*/
private CompiledNode<V> findNode(final CharSequence key) {
CompiledNode<V> current = this.root;
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
@@ -774,6 +945,77 @@ public final class FrequencyTrie<V> {
return current;
}
/**
* Locates the compiled node for the supplied key slice.
*
* @param key already-normalized key storage
* @param offset first character offset
* @param length number of characters to read
* @return compiled node, or {@code null} if the path does not exist
*/
private CompiledNode<V> findNode(final char[] key, final int offset, final int length) {
CompiledNode<V> current = this.root;
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
for (int traversalOffset = offset + length - 1; traversalOffset >= offset; traversalOffset--) {
current = current.findChild(key[traversalOffset]);
if (current == null) {
return null;
}
}
return current;
}
final int endExclusive = offset + length;
for (int traversalOffset = offset; traversalOffset < endExclusive; traversalOffset++) {
current = current.findChild(key[traversalOffset]);
if (current == null) {
return null;
}
}
return current;
}
/**
* Visits node-local values without allocating result containers.
*
* @param node resolved node, or {@code null}
* @param sink value sink
* @param maxResults maximum values to visit
* @return number of visited values
*/
private int visitNode(final CompiledNode<V> node, final EntrySink<? super V> sink, final int maxResults) {
if (node == null) {
return 0;
}
final V[] orderedValues = node.orderedValues();
final int valueCount = Math.min(orderedValues.length, maxResults);
if (valueCount == 0) {
return 0;
}
final int[] orderedCounts = node.orderedCounts();
int visited = 0;
for (int rank = 0; rank < valueCount; rank++) {
visited++;
if (!sink.accept(orderedValues[rank], orderedCounts[rank], rank)) {
break;
}
}
return visited;
}
/**
* Validates visitor maximum result count.
*
* @param maxResults maximum result count
*/
private static void validateMaxResults(final int maxResults) {
if (maxResults < 0) {
throw new IllegalArgumentException("maxResults must be non-negative.");
}
}
/**
* Applies lookup-time case normalization according to persisted metadata.
*
@@ -781,11 +1023,21 @@ public final class FrequencyTrie<V> {
* @return normalized key for trie traversal
*/
private String normalizeLookupKey(final String key) {
return normalizeLookupKey((CharSequence) key).toString();
}
/**
* Applies lookup-time normalization according to persisted metadata.
*
* @param key lookup key
* @return normalized key for trie traversal
*/
private CharSequence normalizeLookupKey(final CharSequence key) {
if (!this.lowercasesLookupKeys && !this.removeDiacritics) {
return key;
}
String normalized = key;
String normalized = key.toString();
if (this.lowercasesLookupKeys) {
normalized = normalized.toLowerCase(Locale.ROOT);
}
@@ -846,9 +1098,9 @@ public final class FrequencyTrie<V> {
/**
* Dense edge lookup span threshold.
* <p>
* This value controls a speed/memory trade-off during freezing:
* dense child lookup tables are allocated only for nodes whose child
* labels fit in this span.
* This value controls a speed/memory trade-off during freezing: dense child
* lookup tables are allocated only for nodes whose child labels fit in this
* span.
* </p>
*/
private final int maxExpandedIndex;
@@ -925,8 +1177,8 @@ public final class FrequencyTrie<V> {
/**
* Creates a new builder with the provided settings, explicit traversal
* direction, explicit case processing mode, explicit diacritic processing
* mode, and an explicit dense child lookup threshold.
* direction, explicit case processing mode, explicit diacritic processing mode,
* and an explicit dense child lookup threshold.
*
* @param arrayFactory array factory
* @param reductionSettings reduction configuration
@@ -934,10 +1186,10 @@ public final class FrequencyTrie<V> {
* @param caseProcessingMode dictionary case processing mode
* @param diacriticProcessingMode dictionary diacritic processing mode
* @param maxExpandedIndex dense lookup span override; zero disables
* dense lookup. Larger values increase direct
* indexing opportunities while potentially
* increasing materialization memory in nodes
* whose edge label span is within the limit.
* dense lookup. Larger values increase direct
* indexing opportunities while potentially
* increasing materialization memory in nodes
* whose edge label span is within the limit.
* @throws NullPointerException if any argument is {@code null}
*/
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
@@ -1052,7 +1304,7 @@ public final class FrequencyTrie<V> {
* @throws IllegalArgumentException if {@code count} is less than {@code 1}
*/
public Builder<V> put(final String key, final V value, final int count) {
Objects.requireNonNull(key, "key");
Objects.requireNonNull(key, ARG_KEY);
Objects.requireNonNull(value, "value");
if (count < 1) { // NOPMD

6
src/main/java/org/egothor/stemmer/FrequencyTrieBuilders.java
View File

@@ -119,11 +119,11 @@ public final class FrequencyTrieBuilders {
* Copies one compiled node and all reachable descendants into the target
* builder.
*
* @param node current compiled node
* @param keyBuilder current key builder
* @param node current compiled node
* @param keyBuilder current key builder
* @param builder target mutable builder
* @param traversalDirection logical key traversal direction used by the source
* @param <V> value type
* @param <V> value type
*/
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {

797
src/main/java/org/egothor/stemmer/PatchCommandEncoder.java
View File

@@ -67,7 +67,7 @@ import java.util.concurrent.locks.ReentrantLock;
* instance can still be used safely when needed.
* </p>
*/
@SuppressWarnings("PMD.CyclomaticComplexity")
@SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" })
public final class PatchCommandEncoder {
/**
@@ -121,6 +121,13 @@ public final class PatchCommandEncoder {
*/
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
/**
* Return value used by
* {@link #applyTo(CharSequence, String, WordTraversalDirection, char[], int, int)}
* when the caller-owned output range is too small for the transformed text.
*/
public static final int APPLY_INSUFFICIENT_CAPACITY = -1;
/**
* Prefix used in unsupported NOOP patch argument exceptions.
*/
@@ -346,6 +353,78 @@ public final class PatchCommandEncoder {
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
}
/**
* Applies a compact patch command into a caller-owned output buffer.
*
* <p>
* The output array is not retained. Capacity failure is reported by
* {@link #APPLY_INSUFFICIENT_CAPACITY} and leaves the output range unchanged.
* Malformed compatibility cases preserve the source exactly as
* {@link #apply(String, String, WordTraversalDirection)} does.
* </p>
*
* @param source original source text
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @param output caller-owned output storage
* @param outputOffset first writable output offset
* @param outputLength writable output capacity
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
* when {@code outputLength} is too small
*/
public static int applyTo(final CharSequence source, final String patchCommand,
final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset,
final int outputLength) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(traversalDirection, "traversalDirection");
Objects.requireNonNull(output, "output");
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
final int sourceLength = source.length();
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
if (producedLength > outputLength) {
return APPLY_INSUFFICIENT_CAPACITY;
}
applyToOutput(source, 0, sourceLength, patchCommand, traversalDirection, output, outputOffset, producedLength);
return producedLength;
}
/**
* Applies a compact patch command from a caller-owned source slice into a
* caller-owned output buffer.
*
* @param source source storage
* @param sourceOffset first source character offset
* @param sourceLength number of source characters
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @param output caller-owned output storage
* @param outputOffset first writable output offset
* @param outputLength writable output capacity
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
* when {@code outputLength} is too small
* @throws IllegalArgumentException when source and output ranges overlap in the
* same array
*/
public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int outputLength) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(traversalDirection, "traversalDirection");
Objects.requireNonNull(output, "output");
Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length);
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength);
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
if (producedLength > outputLength) {
return APPLY_INSUFFICIENT_CAPACITY;
}
applyToOutput(source, sourceOffset, sourceLength, patchCommand, traversalDirection, output, outputOffset,
producedLength);
return producedLength;
}
/**
* Encodes a patch command using the historical backward Egothor semantics.
*
@@ -409,7 +488,6 @@ public final class PatchCommandEncoder {
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
*/
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
private static String applyBackward(final String source, final String patchCommand) {
if (source == null) {
return null;
@@ -435,7 +513,7 @@ public final class PatchCommandEncoder {
int position = result.length() - 1;
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -493,7 +571,6 @@ public final class PatchCommandEncoder {
* @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null}
*/
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
private static String applyForward(final String source, final String patchCommand) {
if (source == null) {
return null;
@@ -519,7 +596,7 @@ public final class PatchCommandEncoder {
int position = 0;
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -681,7 +758,7 @@ public final class PatchCommandEncoder {
*/
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -722,7 +799,7 @@ public final class PatchCommandEncoder {
*/
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
@@ -753,6 +830,711 @@ public final class PatchCommandEncoder {
return result.toString();
}
/**
* Computes the transformed length or the preserved source length for malformed
* compatibility cases.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return produced length
*/
private static int computeAppliedLength(final int sourceLength, final String patchCommand,
final WordTraversalDirection traversalDirection) {
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|| (patchCommand.length() & 1) != 0) {
return sourceLength;
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
return computeBackwardAppliedLength(sourceLength, patchCommand);
}
return computeForwardAppliedLength(sourceLength, patchCommand);
}
/**
* Computes the backward traversal output length.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return produced length
*/
private static int computeBackwardAppliedLength(final int sourceLength, final String patchCommand) {
if (patchCommand.length() == 2) {
return computeSingleBackwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return computeBackwardEmptyAppliedLength(patchCommand);
}
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return sourceLength;
}
position = position - skipCount + 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return sourceLength;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1) {
return sourceLength;
}
final int deleteEndExclusive = position + 1;
position -= deleteCount - 1;
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
return sourceLength;
}
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
if (position < -1 || position >= currentLength) {
return sourceLength;
}
currentLength++;
position++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position--;
}
return currentLength;
}
/**
* Computes the forward traversal output length.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return produced length
*/
private static int computeForwardAppliedLength(final int sourceLength, final String patchCommand) {
if (patchCommand.length() == 2) {
return computeSingleForwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return computeForwardEmptyAppliedLength(patchCommand);
}
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return sourceLength;
}
position = position + skipCount - 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return sourceLength;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
return sourceLength;
}
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
if (position < 0 || position > currentLength) {
return sourceLength;
}
currentLength++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position++;
}
return currentLength;
}
/**
* Computes a single backward instruction output length.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return produced length
*/
private static int computeSingleBackwardAppliedLength(final int sourceLength, final char opcode,
final char argument) {
final int encodedValue;
switch (opcode) {
case DELETE_OPCODE:
encodedValue = decodeEncodedCount(argument);
return encodedValue < 1 || encodedValue > sourceLength ? sourceLength : sourceLength - encodedValue;
case INSERT_OPCODE:
return sourceLength + 1;
case REPLACE_OPCODE:
case SKIP_OPCODE:
return sourceLength;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
/**
* Computes a single forward instruction output length.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return produced length
*/
private static int computeSingleForwardAppliedLength(final int sourceLength, final char opcode,
final char argument) {
return computeSingleBackwardAppliedLength(sourceLength, opcode, argument);
}
/**
* Computes output length for an empty source in backward traversal.
*
* @param patchCommand patch command
* @return produced length
*/
private static int computeBackwardEmptyAppliedLength(final String patchCommand) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case INSERT_OPCODE:
currentLength++;
break;
case SKIP_OPCODE:
case REPLACE_OPCODE:
case DELETE_OPCODE:
return 0;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return 0;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
return currentLength;
}
/**
* Computes output length for an empty source in forward traversal.
*
* @param patchCommand patch command
* @return produced length
*/
private static int computeForwardEmptyAppliedLength(final String patchCommand) {
return computeBackwardEmptyAppliedLength(patchCommand);
}
/**
* Applies an already-sized patch into caller output.
*
* @param source source text
* @param sourceOffset source offset
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @param output output storage
* @param outputOffset output offset
* @param producedLength already-validated produced length
*/
private static void applyToOutput(final CharSequence source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int producedLength) {
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
copySource(source, sourceOffset, sourceLength, output, outputOffset);
return;
}
if (sourceLength > 0) {
copySource(source, sourceOffset, sourceLength, output, outputOffset);
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
} else {
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
}
}
/**
* Applies an already-sized patch into caller output.
*
* @param source source storage
* @param sourceOffset source offset
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @param output output storage
* @param outputOffset output offset
* @param producedLength already-validated produced length
*/
private static void applyToOutput(final char[] source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int producedLength) {
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
return;
}
if (sourceLength > 0) {
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
} else {
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
}
}
/**
* Determines whether the output is exactly the original source.
*
* @param sourceLength source length
* @param producedLength produced length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return {@code true} if copying the source is sufficient
*/
private static boolean isPreservedSource(final int sourceLength, final int producedLength,
final String patchCommand, final WordTraversalDirection traversalDirection) {
return producedLength == sourceLength
&& isKnownPreserveOnlyPatch(sourceLength, patchCommand, traversalDirection);
}
/**
* Returns whether equal length also means no mutation is needed.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return {@code true} when the command preserves source content
*/
private static boolean isKnownPreserveOnlyPatch(final int sourceLength, final String patchCommand,
final WordTraversalDirection traversalDirection) {
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|| (patchCommand.length() & 1) != 0) {
return true;
}
if (patchCommand.length() == 2) {
return isSingleInstructionPreserveOnly(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return hasEmptySourcePreserveOnlyPatch(patchCommand);
}
return traversalDirection == WordTraversalDirection.BACKWARD
? hasBackwardPreserveOnlyPatch(sourceLength, patchCommand)
: hasForwardPreserveOnlyPatch(sourceLength, patchCommand);
}
/**
* Tests whether a single instruction preserves the source content.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return {@code true} when no mutation should be applied
*/
private static boolean isSingleInstructionPreserveOnly(final int sourceLength, final char opcode,
final char argument) {
switch (opcode) {
case DELETE_OPCODE:
final int encodedValue = decodeEncodedCount(argument);
return encodedValue < 1 || encodedValue > sourceLength;
case INSERT_OPCODE:
return false;
case REPLACE_OPCODE:
return sourceLength == 0;
case SKIP_OPCODE:
return true;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
/**
* Tests whether an empty-source patch preserves the source.
*
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasEmptySourcePreserveOnlyPatch(final String patchCommand) {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case INSERT_OPCODE:
break;
case SKIP_OPCODE:
case REPLACE_OPCODE:
case DELETE_OPCODE:
return true;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
return false;
}
/**
* Tests whether a backward patch preserves the source because it is malformed
* or a NOOP.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasBackwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return true;
}
position = position - skipCount + 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return true;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1) {
return true;
}
final int deleteEndExclusive = position + 1;
position -= deleteCount - 1;
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
return true;
}
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
if (position < -1 || position >= currentLength) {
return true;
}
currentLength++;
position++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position--;
}
return false;
}
/**
* Tests whether a forward patch preserves the source because it is malformed or
* a NOOP.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasForwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return true;
}
position = position + skipCount - 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return true;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
return true;
}
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
if (position < 0 || position > currentLength) {
return true;
}
currentLength++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position++;
}
return false;
}
/**
* Copies source characters from a sequence.
*
* @param source source text
* @param sourceOffset source offset
* @param sourceLength source length
* @param output output storage
* @param outputOffset output offset
*/
private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength,
final char[] output, final int outputOffset) {
for (int index = 0; index < sourceLength; index++) {
output[outputOffset + index] = source.charAt(sourceOffset + index);
}
}
/**
* Applies a backward patch after validation.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param output output storage initialized with source
* @param outputOffset output offset
*/
private static void applyBackwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
final int outputOffset) {
if (sourceLength == 0) {
applyBackwardEmptyToOutput(patchCommand, output, outputOffset);
return;
}
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
position = position - decodeEncodedCount(argument) + 1;
break;
case REPLACE_OPCODE:
output[outputOffset + position] = argument;
break;
case DELETE_OPCODE:
final int deleteEndExclusive = position + 1;
position -= decodeEncodedCount(argument) - 1;
System.arraycopy(output, outputOffset + deleteEndExclusive, output, outputOffset + position,
currentLength - deleteEndExclusive);
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
final int insertIndex = position + 1;
System.arraycopy(output, outputOffset + insertIndex, output, outputOffset + insertIndex + 1,
currentLength - insertIndex);
output[outputOffset + insertIndex] = argument;
currentLength++;
position++;
break;
case NOOP_OPCODE:
return;
default:
throw new AssertionError("Patch command was not validated.");
}
position--;
}
}
/**
* Applies a forward patch after validation.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param output output storage initialized with source
* @param outputOffset output offset
*/
private static void applyForwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
final int outputOffset) {
if (sourceLength == 0) {
applyForwardEmptyToOutput(patchCommand, output, outputOffset);
return;
}
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
position = position + decodeEncodedCount(argument) - 1;
break;
case REPLACE_OPCODE:
output[outputOffset + position] = argument;
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
System.arraycopy(output, outputOffset + position + deleteCount, output, outputOffset + position,
currentLength - position - deleteCount);
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
System.arraycopy(output, outputOffset + position, output, outputOffset + position + 1,
currentLength - position);
output[outputOffset + position] = argument;
currentLength++;
break;
case NOOP_OPCODE:
return;
default:
throw new AssertionError("Patch command was not validated.");
}
position++;
}
}
/**
* Applies an empty-source backward patch after validation.
*
* @param patchCommand patch command
* @param output output storage
* @param outputOffset output offset
*/
private static void applyBackwardEmptyToOutput(final String patchCommand, final char[] output,
final int outputOffset) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char argument = patchCommand.charAt(patchIndex + 1);
System.arraycopy(output, outputOffset, output, outputOffset + 1, currentLength);
output[outputOffset] = argument;
currentLength++;
}
}
/**
* Applies an empty-source forward patch after validation.
*
* @param patchCommand patch command
* @param output output storage
* @param outputOffset output offset
*/
private static void applyForwardEmptyToOutput(final String patchCommand, final char[] output,
final int outputOffset) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
output[outputOffset + currentLength] = patchCommand.charAt(patchIndex + 1);
currentLength++;
}
}
/**
* Validates that source and output slices do not overlap when backed by the
* same array.
*
* @param source source storage
* @param sourceOffset source offset
* @param sourceLength source length
* @param output output storage
* @param outputOffset output offset
* @param outputLength output length
*/
private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset,
final int sourceLength, final char[] output, final int outputOffset, final int outputLength) {
if (!source.equals(output) || sourceLength == 0 || outputLength == 0) {
return;
}
final int sourceEnd = sourceOffset + sourceLength;
final int outputEnd = outputOffset + outputLength;
if (sourceOffset < outputEnd && outputOffset < sourceEnd) {
throw new IllegalArgumentException("source and output ranges must not overlap.");
}
}
/**
* Returns the direction-specialized apply strategy.
*
@@ -769,7 +1551,6 @@ public final class PatchCommandEncoder {
* @param argument serialized count argument
* @return decoded positive count, or {@code -1} when the argument is malformed
*/
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
private static int decodeEncodedCount(final char argument) {
if (argument < 'a') {
return -1;

9
src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java
View File

@@ -95,8 +95,8 @@ public final class StemmerPatchTrieBinaryIO {
}
/**
* Reads a GZip-compressed binary patch-command trie from a filesystem path
* with an optional dense child lookup span override.
* Reads a GZip-compressed binary patch-command trie from a filesystem path with
* an optional dense child lookup span override.
* <p>
* This is a runtime-only tuning parameter. The dense-span setting is not
* persisted in the file and does not change the compiled metadata.
@@ -183,14 +183,15 @@ public final class StemmerPatchTrieBinaryIO {
* persisted in the file and does not change the compiled metadata.
* </p>
*
* @param inputStream source stream
* @param inputStream source stream
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return deserialized trie
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final InputStream inputStream, final int maxExpandedIndex) throws IOException {
public static FrequencyTrie<String> read(final InputStream inputStream, final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(inputStream, "inputStream");
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));

5
src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java
View File

@@ -461,7 +461,7 @@ public final class StemmerPatchTrieLoader {
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
throws IOException {
throws IOException {
Objects.requireNonNull(path, PARAMETER_PATH);
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
diacriticProcessingMode);
@@ -816,7 +816,8 @@ public final class StemmerPatchTrieLoader {
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex) throws IOException {
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex);
}

12
src/main/java/org/egothor/stemmer/package-info.java
View File

@@ -58,17 +58,17 @@
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
* traditional line-oriented tab-separated values resource format in which each
* non-empty logical line starts with a canonical stem followed by known surface
* variants in subsequent tab-separated columns.
* Parsing is delegated to {@link org.egothor.stemmer.StemmerDictionaryParser},
* which applies configurable case processing through
* variants in subsequent tab-separated columns. Parsing is delegated to
* {@link org.egothor.stemmer.StemmerDictionaryParser}, which applies
* configurable case processing through
* {@link org.egothor.stemmer.CaseProcessingMode} (default:
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
* supports whole-line as well as trailing remarks introduced by {@code #} or
* {@code //}, and currently ignores dictionary items containing Unicode
* whitespace characters while reporting them through warning-level diagnostics.
* During loading, each variant is converted into a patch command
* targeting the canonical stem, and the stem itself may optionally be stored
* under the canonical no-operation patch.
* During loading, each variant is converted into a patch command targeting the
* canonical stem, and the stem itself may optionally be stored under the
* canonical no-operation patch.
* </p>
*
* <p>

29
src/main/java/org/egothor/stemmer/trie/CompiledNode.java
View File

@@ -48,8 +48,8 @@ import java.util.Objects;
public final class CompiledNode<V> {
/**
* Default dense child lookup span in characters used when an explicit override is
* not provided.
* Default dense child lookup span in characters used when an explicit override
* is not provided.
*/
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
@@ -71,8 +71,8 @@ public final class CompiledNode<V> {
/**
* Dense child lookup table used when labels fit into a compact char interval.
* <p>
* The table enables direct O(1) indexing for child lookup and is allocated
* only when the character span of this node's edges is within the configured
* The table enables direct O(1) indexing for child lookup and is allocated only
* when the character span of this node's edges is within the configured
* threshold.
* </p>
*/
@@ -111,8 +111,8 @@ public final class CompiledNode<V> {
*
* @param maxExpandedIndex upper bound for the dense lookup interval size; zero
* disables dense lookup. Larger values improve
* direct-index likelihood while increasing dense
* table memory in compact-label nodes.
* direct-index likelihood while increasing dense table
* memory in compact-label nodes.
* @throws NullPointerException if any array argument is {@code null}
* @throws IllegalArgumentException if the edge-related arrays or value-related
* arrays do not have matching lengths or the
@@ -288,7 +288,8 @@ public final class CompiledNode<V> {
}
/**
* Returns a small memory-related metric describing this node's dense table size.
* Returns a small memory-related metric describing this node's dense table
* size.
*
* @return number of dense table slots, or {@code 0} when dense lookup is not
* enabled
@@ -328,8 +329,9 @@ public final class CompiledNode<V> {
return false;
}
return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children)
&& Arrays.equals(this.orderedValues, other.orderedValues) && Arrays.equals(this.orderedCounts, other.orderedCounts)
&& this.denseEdgeMin == other.denseEdgeMin && Arrays.equals(this.denseChildren, other.denseChildren);
&& Arrays.equals(this.orderedValues, other.orderedValues)
&& Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin
&& Arrays.equals(this.denseChildren, other.denseChildren);
}
/**
@@ -339,9 +341,8 @@ public final class CompiledNode<V> {
*/
@Override
public String toString() {
return "CompiledNode{"
+ "edgeCount=" + this.edgeLabels.length + ", orderedValueCount=" + this.orderedValues.length
+ ", denseTableLength=" + denseTableLength() + '}';
return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount="
+ this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}';
}
/**
@@ -350,8 +351,8 @@ public final class CompiledNode<V> {
* Lookup order is:
* <ol>
* <li>dense array index (if the label interval is compact enough),</li>
* <li>small-child linear scan when the fallback node has {@value #LINEAR_CHILD_COUNT_THRESHOLD}
* or fewer edges,</li>
* <li>small-child linear scan when the fallback node has
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
* <li>binary search over sorted labels.</li>
* </ol>
* </p>

145
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -45,6 +45,7 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.DisplayName;
@@ -63,6 +64,7 @@ import org.junit.jupiter.api.Test;
@Tag("unit")
@Tag("trie")
@Tag("frequency-trie")
@Tag("lookup")
@DisplayName("FrequencyTrie")
class FrequencyTrieTest {
@@ -398,6 +400,149 @@ class FrequencyTrieTest {
() -> assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1))));
}
/**
* Verifies that visitor lookup returns the same deterministic order and counts
* as the allocating APIs.
*/
@Test
@DisplayName("Visitor lookup matches getAll order and getEntries counts")
void visitorLookupMatchesGetAllOrderAndGetEntriesCounts() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun", 3);
builder.put("house", "verb", 2);
builder.put("house", "adjective", 1);
final FrequencyTrie<String> trie = builder.build();
final List<String> values = new ArrayList<>();
final List<Integer> counts = new ArrayList<>();
final List<Integer> ranks = new ArrayList<>();
final int visited = trie.getAllNormalized("house", (value, count, rank) -> {
values.add(value);
counts.add(count);
ranks.add(rank);
return true;
}, 10);
assertAll(() -> assertEquals(3, visited),
() -> assertEquals(List.of("noun", "verb", "adjective"), values),
() -> assertEquals(List.of(3, 2, 1), counts),
() -> assertEquals(List.of(0, 1, 2), ranks));
}
/**
* Verifies visitor maximum result and early-stop behavior.
*/
@Test
@DisplayName("Visitor lookup honors maxResults and sink early stop")
void visitorLookupHonorsMaxResultsAndSinkEarlyStop() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun", 3);
builder.put("house", "verb", 2);
builder.put("house", "adjective", 1);
final FrequencyTrie<String> trie = builder.build();
final List<String> limited = new ArrayList<>();
final List<String> stopped = new ArrayList<>();
final int limitedCount = trie.getAllNormalized("house", (value, count, rank) -> {
limited.add(value);
return true;
}, 2);
final int stoppedCount = trie.getAllNormalized("house", (value, count, rank) -> {
stopped.add(value);
return false;
}, 10);
assertAll(() -> assertEquals(2, limitedCount),
() -> assertEquals(List.of("noun", "verb"), limited),
() -> assertEquals(1, stoppedCount),
() -> assertEquals(List.of("noun"), stopped));
}
/**
* Verifies visitor zero, negative, missing, and first-result behavior.
*/
@Test
@DisplayName("Visitor lookup handles zero, negative, missing, and first-result cases")
void visitorLookupHandlesBoundaryCases() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun");
final FrequencyTrie<String> trie = builder.build();
final int[] calls = new int[1];
assertAll(() -> assertEquals(0, trie.getAllNormalized("house", (value, count, rank) -> {
calls[0]++;
return true;
}, 0)),
() -> assertEquals(0, calls[0]),
() -> assertThrows(IllegalArgumentException.class,
() -> trie.getAllNormalized("house", (value, count, rank) -> true, -1)),
() -> assertEquals(0, trie.getAllNormalized("missing", (value, count, rank) -> true, 10)),
() -> assertFalse(trie.getFirstNormalized("missing", (value, count, rank) -> true)),
() -> assertTrue(trie.getFirstNormalized("house", (value, count, rank) -> {
assertEquals("noun", value);
assertEquals(1, count);
assertEquals(0, rank);
return true;
})));
}
/**
* Verifies visitor API argument validation.
*/
@Test
@DisplayName("Visitor lookup rejects null and invalid range arguments")
void visitorLookupRejectsNullAndInvalidRangeArguments() {
final FrequencyTrie<String> trie = rankedBuilder().build();
final char[] key = "house".toCharArray();
final FrequencyTrie.EntrySink<String> sink = (value, count, rank) -> true;
assertAll(() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized((char[]) null, 0, 0, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized(key, 0, key.length, null, 1)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> trie.getAllNormalized(key, -1, key.length, sink, 1)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> trie.getAllNormalized(key, 1, key.length, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized("house", null, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAll((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAll("house", null, 1)),
() -> assertThrows(IllegalArgumentException.class,
() -> trie.getAll("house", sink, -1)));
}
/**
* Verifies normalized char-array slices and metadata-aware CharSequence visitor
* lookup.
*/
@Test
@DisplayName("Visitor lookup supports normalized char slices and metadata-aware CharSequence keys")
void visitorLookupSupportsCharSlicesAndMetadataAwareKeys() {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
builder.put("house", "noun");
final FrequencyTrie<String> trie = builder.build();
final char[] padded = "__house__".toCharArray();
assertAll(() -> assertEquals(1,
trie.getAllNormalized(padded, 2, 5, (value, count, rank) -> {
assertEquals("noun", value);
return true;
}, 10)),
() -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true),
"Normalized lookup must bypass metadata lowercasing."),
() -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> {
assertEquals("noun", value);
return true;
})));
}
/**
* Verifies that equal frequencies prefer the shorter string representation.
*/

362
src/test/java/org/egothor/stemmer/PatchCommandEncoderTest.java
View File

@@ -31,6 +31,7 @@
package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -68,6 +69,8 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
@Tag("encoding")
@Tag("apply")
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class PatchCommandEncoderTest {
@@ -147,6 +150,63 @@ class PatchCommandEncoderTest {
Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher"));
}
/**
* Provides explicit forward-direction single-instruction patch application cases.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardSingleInstructionApplyCases() {
return Stream.of(
// 1
Arguments.of(1, "abcd", "Db", "cd"),
// 2
Arguments.of(2, "abc", "Ia", "aabc"),
// 3
Arguments.of(3, "abc", "Ra", "abc"),
// 4
Arguments.of(4, "abc", "-a", "abc"),
// 5
Arguments.of(5, "abc", PatchCommandEncoder.NOOP_PATCH, "abc"));
}
/**
* Provides forward-direction applyTo cases that exercise preserve-only and
* non-preserve branches.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardApplyToCases() {
return Stream.of(
// 1
Arguments.of(1, "book", "-aRa", "baok"),
// 2
Arguments.of(2, "abc", "-dRa", "abc"),
// 3
Arguments.of(3, "abc", "DdRa", "abc"),
// 4
Arguments.of(4, "abc", "-dIa", "abc"),
// 5
Arguments.of(5, "abc", "Na-a", "abc"));
}
/**
* Provides empty-source forward applyTo cases that cover empty-source
* instruction handling.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardEmptySourceApplyCases() {
return Stream.of(
// 1
Arguments.of(1, "IaIb", "ab"),
// 2
Arguments.of(2, "-aRa", ""),
// 3
Arguments.of(3, "IaRa", ""),
// 4
Arguments.of(4, "Na-a", ""));
}
/**
* Provides malformed or index-invalid patch inputs that must preserve the
* original source according to the implementation contract.
@@ -236,12 +296,31 @@ class PatchCommandEncoderTest {
return new StringBuilder(text).reverse().toString();
}
/**
* Applies a patch into a right-sized output buffer and returns the produced
* string.
*
* @param source source text
* @param patch patch command
* @param traversalDirection traversal direction
* @return transformed text
*/
private static String applyToString(final String source, final String patch,
final WordTraversalDirection traversalDirection) {
final char[] output = new char[Math.max(source.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo(source, patch, traversalDirection, output, 0, output.length);
return new String(output, 0, produced);
}
/**
* Tests constructor validation and basic instantiation behavior.
*/
@Nested
@DisplayName("construction")
@Tag("construction")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ConstructionTests {
/**
@@ -327,6 +406,9 @@ class PatchCommandEncoderTest {
@Nested
@DisplayName("encode(String, String)")
@Tag("encoding")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class EncodeTests {
/**
@@ -461,6 +543,9 @@ class PatchCommandEncoderTest {
@Nested
@DisplayName("apply(String, String)")
@Tag("apply")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ApplyTests {
/**
@@ -535,6 +620,101 @@ class PatchCommandEncoderTest {
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
}
/**
* Verifies explicit single-instruction forward patch application
* semantics.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: forward single instruction apply({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardSingleInstructionApplyCases")
@DisplayName("applies forward single instruction patches correctly")
void shouldApplyForwardSingleInstructionsExplicitly(int caseId, String source, String patch, String expected) {
assertEquals(expected, PatchCommandEncoder.apply(source, patch, WordTraversalDirection.FORWARD),
() -> "Case " + caseId + " failed for source='" + source + "', patch='" + patch + "'.");
}
/**
* Verifies forward single-instruction malformed commands fail fast.
*/
@Test
@DisplayName("throws for unsupported forward opcode and NOOP argument")
void shouldThrowForUnsupportedForwardOpcodeAndNoopArgument() {
assertAll(() -> assertEquals("Unsupported patch opcode: X",
assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("abc", "Xa", WordTraversalDirection.FORWARD))
.getMessage()),
() -> assertEquals("Unsupported NOOP patch argument: `",
assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("abc", "N`", WordTraversalDirection.FORWARD))
.getMessage()));
}
/**
* Verifies explicit forward-applyTo cases that exercise preserve-only and
* non-preserve branches.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyToForward({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardApplyToCases")
@DisplayName("applyTo handles forward preserve-only and mutation branches")
void shouldApplyToForwardPreserveAndMutationBranches(int caseId, String source, String patch, String expected) {
final char[] output = new char[Math.max(source.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.FORWARD, output, 0,
output.length);
assertAll(
() -> assertEquals(expected.length(), produced,
() -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 0, produced),
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
}
/**
* Verifies empty-source forward applyTo behavior for insert-only and malformed
* instructions.
*
* @param caseId numeric case identifier
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyToForward(\"\", {1}) -> \"{2}\"")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardEmptySourceApplyCases")
@DisplayName("applies forward empty-source patches correctly")
void shouldApplyToForwardEmptySourceCases(int caseId, String patch, String expected) {
final char[] output = new char[Math.max(expected.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo("", patch, WordTraversalDirection.FORWARD, output, 0,
output.length);
assertAll(
() -> assertEquals(expected.length(), produced,
() -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 0, produced),
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
}
/**
* Verifies malformed empty-source forward patches fail fast and preserve
* empty-source semantics.
*/
@Test
@DisplayName("throws for unsupported NOOP argument on empty-source forward patch")
void shouldThrowForUnsupportedNoopArgumentOnForwardEmptySource() {
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("", "N`Ra", WordTraversalDirection.FORWARD));
assertEquals("Unsupported NOOP patch argument: `", exception.getMessage());
}
/**
* Verifies explicit patch application cases.
*
@@ -590,6 +770,181 @@ class PatchCommandEncoderTest {
assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId
+ " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'.");
}
/**
* Verifies buffer application against string-returning application.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyTo({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideApplyCases")
@DisplayName("applyTo matches apply for explicit backward patch commands")
void shouldApplyToBufferLikeApplyForBackwardCommands(int caseId, String source, String patch, String expected) {
final char[] output = "___..............".toCharArray();
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.BACKWARD, output, 3,
output.length - 3);
assertAll(() -> assertEquals(expected.length(), produced, () -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 3, produced)));
}
/**
* Verifies char-array source slices.
*/
@Test
@DisplayName("applyTo supports char-array source slices")
void shouldApplyToCharArraySourceSlice() {
final char[] source = "__teacher__".toCharArray();
final char[] output = new char[16];
final int produced = PatchCommandEncoder.applyTo(source, 2, 7, "Db", WordTraversalDirection.BACKWARD,
output, 1, output.length - 1);
assertAll(() -> assertEquals(5, produced), () -> assertEquals("teach", new String(output, 1, produced)));
}
/**
* Verifies null and range validation for buffer application.
*/
@Test
@DisplayName("applyTo rejects null and invalid range arguments")
void shouldRejectNullAndInvalidRangeArguments() {
final char[] source = "teacher".toCharArray();
final char[] output = new char[16];
assertAll(() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo((CharSequence) null, "Db", WordTraversalDirection.BACKWARD,
output, 0, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", null, output, 0, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD, null,
0, output.length)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD,
output, -1, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo((char[]) null, 0, 7, "Db",
WordTraversalDirection.BACKWARD, output, 0, output.length)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> PatchCommandEncoder.applyTo(source, 1, source.length, "Db",
WordTraversalDirection.BACKWARD, output, 0, output.length)));
}
/**
* Verifies null, empty, and canonical NOOP patch preservation.
*/
@Test
@DisplayName("applyTo preserves source for null, empty, and canonical NOOP patches")
void shouldApplyToPreserveSourceForEmptyCompatibilityPatches() {
final char[] nullPatchOutput = new char[8];
final char[] emptyPatchOutput = new char[8];
final char[] noopOutput = new char[8];
final int nullPatchLength = PatchCommandEncoder.applyTo("teacher", null, WordTraversalDirection.BACKWARD,
nullPatchOutput, 0, nullPatchOutput.length);
final int emptyPatchLength = PatchCommandEncoder.applyTo("teacher", "", WordTraversalDirection.BACKWARD,
emptyPatchOutput, 0, emptyPatchOutput.length);
final int noopLength = PatchCommandEncoder.applyTo("teacher", PatchCommandEncoder.NOOP_PATCH,
WordTraversalDirection.BACKWARD, noopOutput, 0, noopOutput.length);
assertAll(() -> assertEquals(7, nullPatchLength),
() -> assertEquals("teacher", new String(nullPatchOutput, 0, nullPatchLength)),
() -> assertEquals(7, emptyPatchLength),
() -> assertEquals("teacher", new String(emptyPatchOutput, 0, emptyPatchLength)),
() -> assertEquals(7, noopLength),
() -> assertEquals("teacher", new String(noopOutput, 0, noopLength)));
}
/**
* Verifies insufficient capacity behavior.
*/
@Test
@DisplayName("applyTo reports insufficient capacity without writing output")
void shouldReportInsufficientCapacityWithoutWritingOutput() {
final char[] output = "xxxx".toCharArray();
final int produced = PatchCommandEncoder.applyTo("abc", "Ic", WordTraversalDirection.BACKWARD, output, 0,
output.length - 1);
assertAll(() -> assertEquals(PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY, produced),
() -> assertArrayEquals("xxxx".toCharArray(), output));
}
/**
* Verifies exception parity with string-returning application.
*/
@Test
@DisplayName("applyTo throws for unsupported opcode and NOOP argument")
void shouldApplyToThrowForUnsupportedOpcodeAndNoopArgument() {
final char[] output = new char[8];
assertAll(() -> {
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo("abc", "Xa", WordTraversalDirection.BACKWARD, output, 0,
output.length));
assertEquals("Unsupported patch opcode: X", exception.getMessage());
}, () -> {
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo("abc", "Nb", WordTraversalDirection.BACKWARD, output, 0,
output.length));
assertEquals("Unsupported NOOP patch argument: b", exception.getMessage());
});
}
/**
* Verifies malformed compatibility behavior for buffer application.
*
* @param caseId numeric case identifier
* @param source original source
* @param malformedPatch malformed patch
*/
@ParameterizedTest(name = "[{index}] case {0}: malformed applyTo patch {2} preserves {1}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideMalformedPatchCases")
@DisplayName("applyTo preserves source for malformed or index-invalid patch commands")
void shouldApplyToPreserveSourceForMalformedOrIndexInvalidPatchCommands(int caseId, String source,
String malformedPatch) {
final char[] output = new char[Math.max(source.length(), 1)];
final int produced = PatchCommandEncoder.applyTo(source, malformedPatch, WordTraversalDirection.BACKWARD,
output, 0, output.length);
assertAll(() -> assertEquals(source.length(), produced, () -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(source, new String(output, 0, produced)));
}
/**
* Verifies explicit traversal direction for buffer application.
*/
@Test
@DisplayName("applyTo follows explicit forward traversal direction")
void shouldApplyToWithForwardTraversalDirection() {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
.traversalDirection(WordTraversalDirection.FORWARD)
.build();
final String patch = encoder.encode("cities", "city");
assertEquals(PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD),
applyToString("cities", patch, WordTraversalDirection.FORWARD));
}
/**
* Verifies overlapping source/output slices are rejected.
*/
@Test
@DisplayName("applyTo rejects overlapping char-array source and output ranges")
void shouldRejectOverlappingSourceAndOutputRanges() {
final char[] buffer = "teacher....".toCharArray();
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo(buffer, 0, 7, "Db", WordTraversalDirection.BACKWARD, buffer, 2, 5));
assertEquals("source and output ranges must not overlap.", exception.getMessage());
}
}
/**
@@ -598,6 +953,9 @@ class PatchCommandEncoderTest {
@Nested
@DisplayName("stemming-oriented scenarios")
@Tag("regression")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class StemmingScenarioTests {
/**
@@ -659,6 +1017,9 @@ class PatchCommandEncoderTest {
@Nested
@DisplayName("reversed-word processing")
@Tag("normalization")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ReversedWordProcessingTests {
/**
@@ -743,6 +1104,7 @@ class PatchCommandEncoderTest {
*/
@ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
@Tag("normalization")
@DisplayName("preserves correctness under mirrored input orientation")
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();

87
src/test/java/org/egothor/stemmer/trie/CompiledNodeAndNodeDataTest.java
View File

@@ -46,9 +46,39 @@ import org.junit.jupiter.api.Test;
*/
@Tag("unit")
@Tag("trie")
@Tag("lookup")
@DisplayName("CompiledNode and NodeData")
class CompiledNodeAndNodeDataTest {
/**
* Creates a typed child array for compiled-node tests.
*
* @param length requested array length
* @return typed child array
*/
@SuppressWarnings("unchecked")
private static CompiledNode<String>[] children(final int length) {
return new CompiledNode[length];
}
/**
* Creates an empty child array for leaf compiled-node tests.
*
* @return empty typed child array
*/
private static CompiledNode<String>[] noChildren() {
return children(0);
}
/**
* Creates a leaf node used as a child in lookup tests.
*
* @return leaf node
*/
private static CompiledNode<String> leaf() {
return new CompiledNode<>(new char[0], noChildren(), new String[0], new int[0]);
}
/**
* Verifies that {@link NodeData} rejects mismatched edge-related array lengths.
*/
@@ -99,8 +129,7 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode rejects mismatched edge and child arrays")
void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[0];
final CompiledNode<String>[] children = noChildren();
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0]));
@@ -114,8 +143,7 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode rejects mismatched value arrays")
void compiledNodeShouldRejectMismatchedValueArrays() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[0];
final CompiledNode<String>[] children = noChildren();
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0]));
@@ -131,8 +159,7 @@ class CompiledNodeAndNodeDataTest {
@DisplayName("CompiledNode accessors expose documented backing arrays")
void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() {
final char[] edgeLabels = new char[] { 'a' };
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[1];
final CompiledNode<String>[] children = children(1);
final String[] orderedValues = new String[] { "stem" };
final int[] orderedCounts = new int[] { 5 };
final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts);
@@ -149,12 +176,11 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode can resolve child via dense lookup table")
void compiledNodeUsesDenseLookupForCompactIntervals() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[4];
children[0] = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
children[1] = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
children[2] = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
children[3] = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String>[] children = children(4);
children[0] = leaf();
children[1] = leaf();
children[2] = leaf();
children[3] = leaf();
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'b', 'c', 'd' }, children,
new String[] { "1", "2", "3", "4" }, new int[] { 1, 1, 1, 1 });
@@ -172,12 +198,11 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode resolves child by linear scan for small degree")
void compiledNodeUsesLinearScanForSmallDegree() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[4];
final CompiledNode<String> childA = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childB = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childC = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childD = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String>[] children = children(4);
final CompiledNode<String> childA = leaf();
final CompiledNode<String> childB = leaf();
final CompiledNode<String> childC = leaf();
final CompiledNode<String> childD = leaf();
children[0] = childA;
children[1] = childB;
children[2] = childC;
@@ -200,13 +225,12 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode resolves child by binary search for large degree")
void compiledNodeUsesBinarySearchForLargeDegree() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] children = new CompiledNode[5];
final CompiledNode<String> childA = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childB = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childC = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childD = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String> childE = new CompiledNode<>(new char[0], new CompiledNode[0], new String[0], new int[0]);
final CompiledNode<String>[] children = children(5);
final CompiledNode<String> childA = leaf();
final CompiledNode<String> childB = leaf();
final CompiledNode<String> childC = leaf();
final CompiledNode<String> childD = leaf();
final CompiledNode<String> childE = leaf();
children[0] = childA;
children[1] = childB;
children[2] = childC;
@@ -230,8 +254,7 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode reports leaf, value and edge presence state")
void compiledNodeReportsNodeStateHelpers() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] childless = new CompiledNode[0];
final CompiledNode<String>[] childless = noChildren();
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], childless, new String[0], new int[0]);
assertTrue(leaf.isLeaf());
@@ -239,11 +262,10 @@ class CompiledNodeAndNodeDataTest {
assertFalse(leaf.hasValues());
assertFalse(leaf.hasEdge('a'));
@SuppressWarnings("unchecked")
final CompiledNode<String>[] child = new CompiledNode[1];
final CompiledNode<String>[] child = children(1);
final String[] orderedValues = new String[] { "leaf" };
final int[] orderedCounts = new int[] { 1 };
child[0] = new CompiledNode<>(new char[0], new CompiledNode[0], orderedValues, orderedCounts);
child[0] = new CompiledNode<>(new char[0], noChildren(), orderedValues, orderedCounts);
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a' }, child, orderedValues, orderedCounts);
assertFalse(node.isLeaf());
@@ -260,9 +282,8 @@ class CompiledNodeAndNodeDataTest {
@Test
@DisplayName("CompiledNode equals and hashCode align for identical structure")
void compiledNodeEqualsAndHashCodeAlignForIdenticalStructure() {
@SuppressWarnings("unchecked")
final CompiledNode<String>[] child = new CompiledNode[1];
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], new CompiledNode[0], new String[] { "v" },
final CompiledNode<String>[] child = children(1);
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], noChildren(), new String[] { "v" },
new int[] { 1 });
child[0] = leaf;