Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(loader): materialize compiled patch tries directly

Browse Source
This commit is contained in:
Leo Galambos
2026-07-27 22:41:00 +02:00
parent d2b974e1b8
commit b45e143c84
7 changed files with 1142 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -242,6 +242,18 @@ public final class FrequencyTrie<V> {
return STREAM_VERSION;
}
/**
* Returns whether the supplied metadata identifies a stream format with a
* serialized value table.
*
* @param metadata parsed trie metadata
* @return {@code true} when values are stored in a stream-local table
* @throws NullPointerException if {@code metadata} is {@code null}
*/
/* default */ static boolean usesValueTableFormat(final TrieMetadata metadata) {
return Objects.requireNonNull(metadata, "metadata").formatVersion() >= VALUE_TABLE_VERSION;
}
/**
* Receives trie values during visitor-style lookup.
*
@@ -817,7 +829,37 @@ public final class FrequencyTrie<V> {
*/
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
return CompiledTrieReader.read(inputStream, arrayFactory, valueCodec, maxExpandedIndex);
Objects.requireNonNull(valueCodec, "valueCodec");
return readFromWithMetadata(inputStream, arrayFactory,
(dataInput, metadata) -> valueCodec.read(dataInput), maxExpandedIndex);
}
/**
* Reads a compiled trie while allowing value decoding to use already parsed
* trie metadata.
*
* <p>
* This package-private path materializes the requested final value type during
* the normal graph read. It does not expose reader state or construct an
* intermediate trie with a different value type.
* </p>
*
* @param inputStream source input stream
* @param arrayFactory factory used to create typed value arrays
* @param valueReader metadata-aware value reader
* @param maxExpandedIndex dense lookup span override; zero disables dense
* lookup, negative values use
* {@link #DEFAULT_MAX_EXPANDED_INDEX}
* @param <V> final value type
* @return deserialized compiled trie containing values returned by
* {@code valueReader}
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if reading fails or the binary format is invalid
*/
/* default */ static <V> FrequencyTrie<V> readFromWithMetadata(final InputStream inputStream,
final IntFunction<V[]> arrayFactory, final MetadataValueStreamReader<V> valueReader,
final int maxExpandedIndex) throws IOException {
return CompiledTrieReader.read(inputStream, arrayFactory, valueReader, maxExpandedIndex);
}
/**
@@ -1044,10 +1086,10 @@ public final class FrequencyTrie<V> {
private static final class CompiledTrieReader {
private static <V> FrequencyTrie<V> read(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
final MetadataValueStreamReader<V> valueReader, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(inputStream, "inputStream");
Objects.requireNonNull(arrayFactory, "arrayFactory");
Objects.requireNonNull(valueCodec, "valueCodec");
Objects.requireNonNull(valueReader, "valueReader");
if (maxExpandedIndex < -1) {
throw new IllegalArgumentException("maxExpandedIndex must be >= -1.");
}
@@ -1074,11 +1116,12 @@ public final class FrequencyTrie<V> {
}
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
final V[] valueTable = version >= VALUE_TABLE_VERSION ? readValueTable(dataInput, arrayFactory, valueCodec)
final V[] valueTable = version >= VALUE_TABLE_VERSION
? readValueTable(dataInput, arrayFactory, valueReader, sourceMetadata)
: null;
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, valueTable, nodeCount,
effectiveMaxExpandedIndex, version);
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueReader, sourceMetadata, valueTable,
nodeCount, effectiveMaxExpandedIndex, version);
final CompiledNode<V> rootNode = nodes[rootNodeId];
if (LOGGER.isLoggable(Level.FINE)) {
@@ -1099,13 +1142,14 @@ public final class FrequencyTrie<V> {
*
* @param dataInput input stream
* @param arrayFactory typed-array factory
* @param valueCodec codec responsible for value decoding
* @param valueReader metadata-aware value reader
* @param metadata parsed trie metadata
* @param <V> value type
* @return decoded values in table-index order
* @throws IOException if the count is negative or value decoding fails
*/
private static <V> V[] readValueTable(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
final ValueStreamCodec<V> valueCodec) throws IOException {
final MetadataValueStreamReader<V> valueReader, final TrieMetadata metadata) throws IOException {
final int distinctValueCount = dataInput.readInt();
if (distinctValueCount < 0) {
throw new IOException("Negative distinct value count: " + distinctValueCount);
@@ -1113,7 +1157,7 @@ public final class FrequencyTrie<V> {
final V[] valueTable = arrayFactory.apply(distinctValueCount);
for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) {
valueTable[valueIndex] = valueCodec.read(dataInput);
valueTable[valueIndex] = valueReader.read(dataInput, metadata);
}
return valueTable;
}
@@ -1181,8 +1225,9 @@ public final class FrequencyTrie<V> {
}
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final V[] valueTable,
final int nodeCount, final int maxExpandedIndex, final int version) throws IOException {
final IntFunction<V[]> arrayFactory, final MetadataValueStreamReader<V> valueReader,
final TrieMetadata metadata, final V[] valueTable, final int nodeCount, final int maxExpandedIndex,
final int version) throws IOException {
final char[][] edgeLabelsByNode = new char[nodeCount][];
final int[][] childNodeIdsByNode = new int[nodeCount][];
@SuppressWarnings("unchecked")
@@ -1234,7 +1279,7 @@ public final class FrequencyTrie<V> {
}
orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex];
} else {
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
orderedValuesByNode[nodeIndex][valueIndex] = valueReader.read(dataInput, metadata);
}
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
@@ -1980,6 +2025,32 @@ public final class FrequencyTrie<V> {
}
}
/**
* Reads one final trie value using metadata parsed from the same binary stream.
*
* <p>
* Implementations are invoked only during deserialization and are not retained
* by the resulting trie. Returned values are stored directly in final compiled
* node arrays.
* </p>
*
* @param <V> final value type
*/
/* default */
@FunctionalInterface
interface MetadataValueStreamReader<V> {
/**
* Reads and materializes one final value.
*
* @param dataInput source data input
* @param metadata already parsed trie metadata
* @return final value to store directly in compiled nodes
* @throws IOException if reading or materialization fails
*/
V read(DataInputStream dataInput, TrieMetadata metadata) throws IOException;
}
/**
* Codec used to persist values stored in the trie.
*

272
src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java
View File

@@ -39,7 +39,10 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
@@ -53,6 +56,8 @@ import java.util.zip.GZIPOutputStream;
* patch commands represented as {@link String}. The serialized trie payload is
* the native binary format of {@link FrequencyTrie}, wrapped in GZip
* compression.
* Binary reads can either preserve those serialized strings or materialize
* {@link CompiledPatchCommand} values directly in the final trie nodes.
*
* <p>
* The helper centralizes the codec and compression details so that higher-level
@@ -71,6 +76,21 @@ public final class StemmerPatchTrieBinaryIO {
*/
private static final FrequencyTrie.ValueStreamCodec<String> STRING_CODEC = new StringValueStreamCodec();
/**
* Maximum serialized patch-command length included in validation diagnostics.
*/
private static final int MAX_DIAGNOSTIC_PATCH_LENGTH = 128;
/**
* Null-check parameter name for filesystem paths.
*/
private static final String PATH_PARAMETER = "path";
/**
* Null-check parameter name for filesystem path strings.
*/
private static final String FILE_NAME_PARAMETER = "fileName";
/**
* Utility class.
*/
@@ -87,7 +107,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final Path path) throws IOException {
Objects.requireNonNull(path, "path");
Objects.requireNonNull(path, PATH_PARAMETER);
try (InputStream fileInputStream = Files.newInputStream(path)) {
return read(fileInputStream);
@@ -110,7 +130,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(path, "path");
Objects.requireNonNull(path, PATH_PARAMETER);
try (InputStream fileInputStream = Files.newInputStream(path)) {
return read(fileInputStream, maxExpandedIndex);
@@ -127,7 +147,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final String fileName) throws IOException {
Objects.requireNonNull(fileName, "fileName");
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
return read(Path.of(fileName));
}
@@ -147,7 +167,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(fileName, "fileName");
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
return read(Path.of(fileName), maxExpandedIndex);
}
@@ -204,6 +224,149 @@ public final class StemmerPatchTrieBinaryIO {
}
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* a filesystem path.
*
* @param path source file
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final Path path) throws IOException {
Objects.requireNonNull(path, PATH_PARAMETER);
try (InputStream fileInputStream = Files.newInputStream(path)) {
return readCompiled(fileInputStream);
}
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* a filesystem path with a dense child lookup span override.
*
* @param path source file
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final Path path,
final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(path, PATH_PARAMETER);
try (InputStream fileInputStream = Files.newInputStream(path)) {
return readCompiled(fileInputStream, maxExpandedIndex);
}
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* a filesystem path string.
*
* @param fileName source file name or path string
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final String fileName) throws IOException {
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
return readCompiled(Path.of(fileName));
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* a filesystem path string with a dense child lookup span override.
*
* @param fileName source file name or path string
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final String fileName,
final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
return readCompiled(Path.of(fileName), maxExpandedIndex);
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* an input stream.
*
* @param inputStream source stream
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream)
throws IOException {
return readCompiled(inputStream, -1);
}
/**
* Reads a compressed binary patch-command trie directly as compiled values from
* an input stream with a dense child lookup span override.
*
* @param inputStream source stream
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream,
final int maxExpandedIndex) throws IOException {
return readCompiled(inputStream, maxExpandedIndex, CompiledPatchCommand::compile);
}
/**
* Reads a compressed binary patch-command trie using a caller-supplied command
* compiler.
*
* <p>
* This package-private seam permits deterministic compilation-count testing
* without global counters. Production callers use
* {@link CompiledPatchCommand#compile(String, WordTraversalDirection)}.
* </p>
*
* @param inputStream source stream
* @param maxExpandedIndex dense lookup span override
* @param commandCompiler compiler for one serialized command and traversal
* direction
* @return directly materialized compiled patch-command trie
* @throws NullPointerException if any argument is {@code null}
* @throws IOException if reading, decompression, or command compilation
* fails
*/
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream,
final int maxExpandedIndex,
final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler)
throws IOException {
Objects.requireNonNull(inputStream, "inputStream");
Objects.requireNonNull(commandCompiler, "commandCompiler");
final CompiledPatchValueReader valueReader = new CompiledPatchValueReader(commandCompiler);
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));
DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) {
final FrequencyTrie<CompiledPatchCommand> trie = FrequencyTrie.readFromWithMetadata(dataInputStream,
CompiledPatchCommand[]::new, valueReader, maxExpandedIndex);
LOGGER.log(Level.FINE, "Read compressed binary stemmer trie directly as compiled patch commands.");
return trie;
}
}
/**
* Reads only metadata from a GZip-compressed binary patch-command trie stored
* at a filesystem path.
@@ -214,7 +377,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static TrieMetadata readMetadata(final Path path) throws IOException {
Objects.requireNonNull(path, "path");
Objects.requireNonNull(path, PATH_PARAMETER);
return read(path).metadata();
}
@@ -228,7 +391,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if reading or decompression fails
*/
public static TrieMetadata readMetadata(final String fileName) throws IOException {
Objects.requireNonNull(fileName, "fileName");
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
return readMetadata(Path.of(fileName));
}
@@ -256,7 +419,7 @@ public final class StemmerPatchTrieBinaryIO {
*/
public static void write(final FrequencyTrie<String> trie, final Path path) throws IOException {
Objects.requireNonNull(trie, "trie");
Objects.requireNonNull(path, "path");
Objects.requireNonNull(path, PATH_PARAMETER);
final Path parent = path.toAbsolutePath().getParent();
if (parent != null) {
@@ -278,7 +441,7 @@ public final class StemmerPatchTrieBinaryIO {
* @throws IOException if writing fails
*/
public static void write(final FrequencyTrie<String> trie, final String fileName) throws IOException {
Objects.requireNonNull(fileName, "fileName");
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
write(trie, Path.of(fileName));
}
@@ -323,4 +486,97 @@ public final class StemmerPatchTrieBinaryIO {
return dataInput.readUTF();
}
}
/**
* Metadata-aware reader that compiles serialized patch commands into final trie
* values.
*
* <p>
* Version 7 value tables already contain distinct serialized values, so each
* entry is compiled directly. Historical inline formats use the reader-local
* equality cache to compile repeated serialized commands once. Neither the cache
* nor this reader is retained after trie loading.
* </p>
*/
private static final class CompiledPatchValueReader
implements FrequencyTrie.MetadataValueStreamReader<CompiledPatchCommand> {
/**
* Compiler used to materialize one final command.
*/
private final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler;
/**
* Compatibility cache used only by historical inline-value streams.
*/
private final Map<String, CompiledPatchCommand> legacyCompiledCommands = new HashMap<>();
/**
* Creates one reader with a caller-supplied compiler.
*
* @param commandCompiler compiler for serialized patch commands
*/
private CompiledPatchValueReader(
final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler) {
this.commandCompiler = commandCompiler;
}
/**
* Reads and compiles one serialized patch command.
*
* @param dataInput source data input
* @param metadata parsed trie metadata
* @return final compiled patch command
* @throws IOException if reading or command compilation fails
*/
@Override
public CompiledPatchCommand read(final DataInputStream dataInput, final TrieMetadata metadata)
throws IOException {
final String serializedPatch = dataInput.readUTF();
if (FrequencyTrie.usesValueTableFormat(metadata)) {
return compile(serializedPatch, metadata.traversalDirection());
}
final CompiledPatchCommand cachedCommand = this.legacyCompiledCommands.get(serializedPatch);
if (cachedCommand != null) {
return cachedCommand;
}
final CompiledPatchCommand compiledCommand = compile(serializedPatch, metadata.traversalDirection());
this.legacyCompiledCommands.put(serializedPatch, compiledCommand);
return compiledCommand;
}
/**
* Compiles one serialized command and converts validation failures to
* trust-boundary {@link IOException} instances.
*
* @param serializedPatch serialized patch command
* @param traversalDirection traversal direction from persisted metadata
* @return compiled patch command
* @throws IOException if the serialized command is invalid
*/
private CompiledPatchCommand compile(final String serializedPatch,
final WordTraversalDirection traversalDirection) throws IOException {
try {
return this.commandCompiler.apply(serializedPatch, traversalDirection);
} catch (IllegalArgumentException exception) {
throw new IOException("Invalid persisted patch command '" + boundedPatch(serializedPatch)
+ "' for traversal direction " + traversalDirection + '.', exception);
}
}
}
/**
* Returns a safely bounded patch-command representation for diagnostics.
*
* @param serializedPatch serialized patch command
* @return complete or length-bounded diagnostic representation
*/
private static String boundedPatch(final String serializedPatch) {
if (serializedPatch.length() <= MAX_DIAGNOSTIC_PATCH_LENGTH) {
return serializedPatch;
}
return serializedPatch.substring(0, MAX_DIAGNOSTIC_PATCH_LENGTH) + "... (length "
+ serializedPatch.length() + ')';
}
}

42
src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java
View File

@@ -1311,6 +1311,12 @@ public final class StemmerPatchTrieLoader {
* Loads a GZip-compressed binary patch-command trie from a filesystem path and
* returns runtime-specialized compiled patch values.
*
* <p>
* Serialized patch commands are compiled while the binary graph is read, so the
* returned trie is materialized directly without an intermediate
* {@code FrequencyTrie<String>} or a second node graph.
* </p>
*
* @param path path to the compressed binary trie file
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code path} is {@code null}
@@ -1318,7 +1324,8 @@ public final class StemmerPatchTrieLoader {
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path) throws IOException {
return compilePatchTrie(loadBinary(path));
Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.readCompiled(path);
}
/**
@@ -1351,6 +1358,11 @@ public final class StemmerPatchTrieLoader {
* a custom dense lookup span override and returns runtime-specialized compiled
* patch values.
*
* <p>
* Serialized patch commands are compiled directly into the final node graph.
* The dense lookup override is applied during that graph materialization.
* </p>
*
* @param path path to the compressed binary trie file
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
@@ -1361,7 +1373,8 @@ public final class StemmerPatchTrieLoader {
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path,
final int maxExpandedIndex) throws IOException {
return compilePatchTrie(loadBinary(path, maxExpandedIndex));
Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex);
}
/**
@@ -1387,6 +1400,11 @@ public final class StemmerPatchTrieLoader {
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* string and returns runtime-specialized compiled patch values.
*
* <p>
* Serialized patch commands are compiled while the binary graph is read, so no
* intermediate String-valued trie is constructed.
* </p>
*
* @param fileName file name or path string
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code fileName} is {@code null}
@@ -1394,7 +1412,8 @@ public final class StemmerPatchTrieLoader {
* read
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName) throws IOException {
return compilePatchTrie(loadBinary(fileName));
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.readCompiled(fileName);
}
/**
@@ -1428,6 +1447,11 @@ public final class StemmerPatchTrieLoader {
* using a custom dense lookup span override and returns runtime-specialized
* compiled patch values.
*
* <p>
* Serialized patch commands are compiled directly into the final node graph.
* The dense lookup override is applied during that graph materialization.
* </p>
*
* @param fileName file name or path string
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
@@ -1438,7 +1462,8 @@ public final class StemmerPatchTrieLoader {
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName,
final int maxExpandedIndex) throws IOException {
return compilePatchTrie(loadBinary(fileName, maxExpandedIndex));
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex);
}
/**
@@ -1462,6 +1487,12 @@ public final class StemmerPatchTrieLoader {
* Loads a GZip-compressed binary patch-command trie from an input stream and
* returns runtime-specialized compiled patch values.
*
* <p>
* Serialized patch commands are compiled while the binary graph is read, so the
* returned trie is materialized directly without an intermediate
* {@code FrequencyTrie<String>} or graph-mapping pass.
* </p>
*
* @param inputStream source input stream
* @return compiled patch-command trie with runtime-specialized values
* @throws NullPointerException if {@code inputStream} is {@code null}
@@ -1469,7 +1500,8 @@ public final class StemmerPatchTrieLoader {
*/
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final InputStream inputStream)
throws IOException {
return compilePatchTrie(loadBinary(inputStream));
Objects.requireNonNull(inputStream, "inputStream");
return StemmerPatchTrieBinaryIO.readCompiled(inputStream);
}
/**

48
src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java
View File

@@ -150,6 +150,25 @@ final class CompiledTrieArtifactRegressionTest {
() -> assertGoldenArtifactSemanticProbes(trie, artifactCase));
}
/**
* Verifies that every committed version 6 artifact can be materialized directly
* as compiled commands while preserving representative stemming behavior.
*
* @param artifactCase regression case
* @throws IOException if artifact loading fails
*/
@ParameterizedTest(name = "{0}")
@MethodSource("artifactCases")
@DisplayName("Committed golden artifacts must load directly as compiled commands")
void shouldLoadGoldenArtifactsDirectlyAsCompiledCommands(final ArtifactCase artifactCase) throws IOException {
final byte[] goldenArtifactBytes = RegressionArtifactSupport
.readResourceBytes(artifactCase.goldenArtifactResource());
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader
.loadBinaryCompiled(new ByteArrayInputStream(goldenArtifactBytes));
assertGoldenCompiledArtifactSemanticProbes(trie, artifactCase);
}
/**
* Verifies in-process determinism independently of the checked-in golden file
* by compiling the same dictionary twice and requiring identical artifact
@@ -264,6 +283,35 @@ final class CompiledTrieArtifactRegressionTest {
}
}
/**
* Verifies representative semantic probes against a directly materialized
* compiled-command trie.
*
* @param trie compiled-command trie to inspect
* @param artifactCase regression case providing the expected probes
*/
private static void assertGoldenCompiledArtifactSemanticProbes(
final FrequencyTrie<CompiledPatchCommand> trie, final ArtifactCase artifactCase) {
for (ProbeExpectation probe : artifactCase.probes()) {
final CompiledPatchCommand[] allPatchCommands = trie.getAll(probe.word());
final CompiledPatchCommand preferredPatchCommand = trie.get(probe.word());
final String preferredStem = preferredPatchCommand == null ? null : preferredPatchCommand.apply(probe.word());
final Set<String> allStems = new LinkedHashSet<String>();
for (CompiledPatchCommand patchCommand : allPatchCommands) {
allStems.add(patchCommand.apply(probe.word()));
}
assertAll(
() -> assertFalse(allPatchCommands.length == 0,
"Representative probe must produce at least one result for word: " + probe.word()),
() -> assertEquals(probe.preferredStem(), preferredStem,
"Preferred stem mismatch for representative probe word: " + probe.word()),
() -> assertTrue(allStems.containsAll(probe.acceptableStems()),
"All acceptable stems must be present in getAll() for representative probe word: "
+ probe.word()));
}
}
/**
* Immutable regression case definition.
*

112
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -1112,6 +1112,82 @@ class FrequencyTrieTest {
() -> assertNull(restored.get("missing")));
}
/**
* Verifies that metadata-aware version 7 reads receive parsed metadata before
* decoding and materialize shared final values directly in compiled nodes.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Metadata-aware version 7 reader materializes shared final values")
void metadataAwareVersionSevenReaderMaterializesSharedFinalValues() throws IOException {
final FrequencyTrie<String> original = sharedValueTrie();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
original.writeTo(outputStream, STRING_CODEC);
final AtomicInteger readCount = new AtomicInteger();
final AtomicInteger observedMetadataVersion = new AtomicInteger();
final FrequencyTrie<StringBuilder> restored = FrequencyTrie.readFromWithMetadata(
new ByteArrayInputStream(outputStream.toByteArray()), StringBuilder[]::new,
(dataInput, metadata) -> {
observedMetadataVersion.set(metadata.formatVersion());
readCount.incrementAndGet();
return new StringBuilder(dataInput.readUTF());
}, -1);
final CompiledNode<StringBuilder> suffixBNode = restored.root().findChild('b');
final CompiledNode<StringBuilder> abNode = suffixBNode.findChild('a');
final CompiledNode<StringBuilder> cbNode = suffixBNode.findChild('c');
assertAll(() -> assertEquals(7, observedMetadataVersion.get()),
() -> assertEquals(3, readCount.get()),
() -> assertSame(restored.get("ab"), restored.get("cb")),
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
() -> assertEquals(StringBuilder[].class, abNode.orderedValues().getClass()),
() -> assertEquals("left", restored.get("xab").toString()),
() -> assertEquals("right", restored.get("ycb").toString()));
}
/**
* Verifies that metadata-aware reading preserves the inline value layout used
* by every historical stream version from 1 through 6.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Metadata-aware reader supports inline values in versions 1 through 6")
void metadataAwareReaderSupportsInlineValuesInVersionsOneThroughSix() throws IOException {
for (int version = 1; version <= 6; version++) {
final int historicalVersion = version;
final byte[] bytes = createSerializedStream(0x45475452, historicalVersion, 1, 0,
dataOutput -> writeMetadataForHistoricalVersion(dataOutput, historicalVersion),
new NodeWriter[] { dataOutput -> {
if (historicalVersion >= 6) {
dataOutput.writeBoolean(false);
}
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeUTF("inline-" + historicalVersion);
dataOutput.writeInt(1);
} });
final AtomicInteger readCount = new AtomicInteger();
final AtomicInteger observedMetadataVersion = new AtomicInteger();
final FrequencyTrie<StringBuilder> trie = FrequencyTrie.readFromWithMetadata(
new ByteArrayInputStream(bytes), StringBuilder[]::new,
(dataInput, metadata) -> {
observedMetadataVersion.set(metadata.formatVersion());
readCount.incrementAndGet();
return new StringBuilder(dataInput.readUTF());
}, -1);
assertAll(() -> assertEquals(historicalVersion, observedMetadataVersion.get()),
() -> assertEquals(1, readCount.get()),
() -> assertEquals("inline-" + historicalVersion, trie.get("").toString()));
}
}
/**
* Verifies fingerprint stability and sensitivity to metadata and trie content.
*/
@@ -2325,6 +2401,42 @@ class FrequencyTrieTest {
dataOutput.writeInt(occurrenceCount);
}
/**
* Writes the metadata layout used by one historical stream version.
*
* @param dataOutput output stream
* @param version historical stream version from 1 through 6
* @throws IOException if writing fails
*/
private static void writeMetadataForHistoricalVersion(final DataOutputStream dataOutput, final int version)
throws IOException {
if (version == 1) {
return;
}
if (version == 2) {
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
return;
}
final ReductionSettings reductionSettings = ReductionSettings
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
if (version <= 4) {
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal());
if (version == 4) {
dataOutput.writeInt(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT.ordinal());
}
return;
}
final TrieMetadata metadata = new TrieMetadata(version, WordTraversalDirection.BACKWARD, reductionSettings,
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
dataOutput.writeUTF(metadata.toTextBlock());
}
/**
* Writes one synthetic metadata block.
*/

334
src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java
View File

@@ -32,6 +32,7 @@ package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -50,14 +51,18 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.egothor.stemmer.trie.CompiledNode;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Tag;
@@ -98,6 +103,17 @@ import org.mockito.MockedStatic;
@DisplayName("StemmerPatchTrieBinaryIO")
class StemmerPatchTrieBinaryIOTest {
/**
* Maximum invalid patch-command length retained verbatim in production
* diagnostics.
*/
private static final int DIAGNOSTIC_PATCH_BOUNDARY = 128;
/**
* Marker that introduces the original length after diagnostic truncation.
*/
private static final String DIAGNOSTIC_TRUNCATION_MARKER = "... (length ";
/**
* Temporary directory provided by JUnit.
*/
@@ -323,7 +339,202 @@ class StemmerPatchTrieBinaryIOTest {
"read(InputStream) must reject null input stream."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"read(InputStream, int) must reject null input stream."));
"read(InputStream, int) must reject null input stream."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((Path) null),
"readCompiled(Path) must reject null path."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((Path) null, 0),
"readCompiled(Path, int) must reject null path."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((String) null),
"readCompiled(String) must reject null file name."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((String) null, 0),
"readCompiled(String, int) must reject null file name."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null),
"readCompiled(InputStream) must reject null input stream."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null, 0),
"readCompiled(InputStream, int) must reject null input stream."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(new byte[0]), 0,
null),
"Injected compiled read must reject a null command compiler."));
}
/**
* Verifies that direct compiled reading stores compiled commands in final
* nodes and shares one version 7 table object across repeated slots.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@DisplayName("Should materialize version 7 directly as shared compiled commands")
void shouldMaterializeVersionSevenDirectlyAsSharedCompiledCommands() throws IOException {
final FrequencyTrie<String> sourceTrie = sharedPatchTrie();
final byte[] artifactBytes = writeCompressed(sourceTrie);
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO
.readCompiled(new ByteArrayInputStream(artifactBytes));
final FrequencyTrie<String> stringTrie = StemmerPatchTrieBinaryIO
.read(new ByteArrayInputStream(artifactBytes));
final CompiledNode<CompiledPatchCommand> suffixBNode = compiledTrie.root().findChild('b');
final CompiledNode<CompiledPatchCommand> abNode = suffixBNode.findChild('a');
final CompiledNode<CompiledPatchCommand> cbNode = suffixBNode.findChild('c');
assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()),
() -> assertEquals(sourceTrie.size(), compiledTrie.size()),
() -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()),
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
() -> assertEquals("a", compiledTrie.get("ab").apply("ab")),
() -> assertEquals("c", compiledTrie.get("cb").apply("cb")),
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")),
() -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")),
() -> assertInstanceOf(String.class, stringTrie.get("ab")),
() -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab")));
}
/**
* Verifies that version 7 direct loading compiles once per distinct value
* table entry.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@DisplayName("Should compile each version 7 table entry once")
void shouldCompileEachVersionSevenTableEntryOnce() throws IOException {
final FrequencyTrie<String> sourceTrie = sharedPatchTrie();
final byte[] artifactBytes = writeCompressed(sourceTrie);
final AtomicInteger compilationCount = new AtomicInteger();
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO.readCompiled(
new ByteArrayInputStream(artifactBytes), -1, (serializedPatch, traversalDirection) -> {
compilationCount.incrementAndGet();
return CompiledPatchCommand.compile(serializedPatch, traversalDirection);
});
assertAll(() -> assertEquals(2, compilationCount.get()),
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")));
}
/**
* Verifies that the direct historical reader compiles repeated inline
* commands once through its reader-local compatibility cache.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@DisplayName("Should compile repeated version 6 inline commands once")
void shouldCompileRepeatedVersionSixInlineCommandsOnce() throws IOException {
final String serializedPatch = PatchCommandEncoder.builder().build().encode("ab", "a");
final byte[] artifactBytes = createVersionSixArtifactWithRepeatedPatch(serializedPatch);
final AtomicInteger compilationCount = new AtomicInteger();
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO.readCompiled(
new ByteArrayInputStream(artifactBytes), -1, (patch, traversalDirection) -> {
compilationCount.incrementAndGet();
return CompiledPatchCommand.compile(patch, traversalDirection);
});
final CompiledPatchCommand rootCommand = compiledTrie.root().orderedValues()[0];
final CompiledPatchCommand childCommand = compiledTrie.root().findChild('a').orderedValues()[0];
assertAll(() -> assertEquals(1, compilationCount.get()),
() -> assertSame(rootCommand, childCommand),
() -> assertEquals(6, compiledTrie.metadata().formatVersion()));
}
/**
* Verifies that invalid persisted patch commands are rejected only by the
* direct compiled path and retain their validation cause.
*
* @throws IOException if test setup I/O fails unexpectedly
*/
@Test
@DisplayName("Should wrap invalid compiled patch commands as IOException")
void shouldWrapInvalidCompiledPatchCommandsAsIOException() throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("invalid", "Zz");
final byte[] artifactBytes = writeCompressed(builder.build());
final FrequencyTrie<String> stringTrie = StemmerPatchTrieBinaryIO
.read(new ByteArrayInputStream(artifactBytes));
final IOException exception = assertThrows(IOException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes)));
assertAll(() -> assertEquals("Zz", stringTrie.get("invalid")),
() -> assertTrue(exception.getMessage().contains("Zz")),
() -> assertTrue(exception.getMessage().contains("BACKWARD")),
() -> assertInstanceOf(IllegalArgumentException.class, exception.getCause()));
}
/**
* Verifies that an invalid command exactly at the diagnostic boundary remains
* complete and does not receive a truncation marker.
*
* @throws IOException if test setup I/O fails unexpectedly
*/
@Test
@DisplayName("Should retain an invalid command exactly at the diagnostic boundary")
void shouldRetainInvalidCommandAtDiagnosticBoundary() throws IOException {
final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY);
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, invalidPatch);
assertAll(() -> assertTrue(exception.getMessage().contains(invalidPatch),
"The exact-boundary command must remain complete."),
() -> assertFalse(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
"The exact-boundary command must not be marked as truncated."));
}
/**
* Verifies that an invalid command one character beyond the diagnostic
* boundary is truncated at the boundary and reports its original length.
*
* @throws IOException if test setup I/O fails unexpectedly
*/
@Test
@DisplayName("Should truncate an invalid command immediately above the diagnostic boundary")
void shouldTruncateInvalidCommandAboveDiagnosticBoundary() throws IOException {
final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY + 1);
final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY)
+ DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')';
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic);
assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
"The boundary-plus-one command must be marked as truncated."),
() -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"),
"The complete boundary-plus-one command must not appear in the diagnostic."));
}
/**
* Verifies that substantially oversized invalid commands produce bounded
* diagnostics containing only the retained prefix, truncation marker, and
* original length.
*
* @throws IOException if test setup I/O fails unexpectedly
*/
@Test
@DisplayName("Should bound diagnostics for substantially oversized invalid commands")
void shouldBoundDiagnosticForOversizedInvalidCommand() throws IOException {
final String invalidPatch = "Z".repeat(512);
final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY)
+ DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')';
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic);
assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
"The oversized command must be marked as truncated."),
() -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"),
"The complete oversized command must not appear in the diagnostic."),
() -> assertTrue(exception.getMessage().length() < invalidPatch.length(),
"The diagnostic must remain shorter than the oversized persisted command."));
}
/**
@@ -614,6 +825,127 @@ class StemmerPatchTrieBinaryIOTest {
}
}
/**
* Builds a version 7 trie with equal patch strings on distinct nonmergeable
* nodes and one additional distinct patch.
*
* @return representative patch-command trie
*/
private static FrequencyTrie<String> sharedPatchTrie() {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
final String sharedPatch = encoder.encode("ab", "a");
final String longerDeletionPatch = encoder.encode("xab", "x");
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("ab", new String(sharedPatch), 2);
builder.put("xab", longerDeletionPatch);
builder.put("cb", new String(sharedPatch), 3);
builder.put("ycb", new String(longerDeletionPatch));
return builder.build();
}
/**
* Serializes one String-valued trie through the production compressed writer.
*
* @param trie source trie
* @return compressed artifact bytes
* @throws IOException if writing fails
*/
private static byte[] writeCompressed(final FrequencyTrie<String> trie) throws IOException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
StemmerPatchTrieBinaryIO.write(trie, outputStream);
return outputStream.toByteArray();
}
/**
* Loads an artifact through an injected rejecting compiler and verifies the
* exact bounded trust-boundary diagnostic independently of patch-parser
* compatibility behavior for odd command lengths.
*
* @param invalidPatch invalid serialized patch command
* @param expectedDiagnostic complete or bounded command representation expected
* inside the diagnostic
* @return thrown I/O exception for additional boundary-specific assertions
* @throws IOException if fixture serialization fails unexpectedly
*/
private static IOException assertInvalidCompiledPatchDiagnostic(final String invalidPatch,
final String expectedDiagnostic) throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("invalid", invalidPatch);
final byte[] artifactBytes = writeCompressed(builder.build());
final IOException exception = assertThrows(IOException.class,
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes), -1,
(serializedPatch, traversalDirection) -> {
throw new IllegalArgumentException("Injected invalid persisted command.");
}),
"Direct compiled loading must translate an invalid persisted command to IOException.");
final String expectedMessage = "Invalid persisted patch command '" + expectedDiagnostic
+ "' for traversal direction BACKWARD.";
assertAll(() -> assertEquals(IOException.class, exception.getClass(),
"IOException must be the exact externally visible exception type."),
() -> assertEquals(expectedMessage, exception.getMessage(),
"The invalid-command diagnostic must match the bounded production contract."),
() -> assertNotNull(exception.getCause(), "The validation cause must be retained."),
() -> assertEquals(IllegalArgumentException.class, exception.getCause().getClass(),
"The original patch-command validation failure must remain the cause."));
return exception;
}
/**
* Creates a valid version 6 artifact containing the same inline patch command
* in two distinct node slots.
*
* @param serializedPatch repeated serialized patch command
* @return compressed historical artifact bytes
* @throws IOException if fixture creation fails
*/
private static byte[] createVersionSixArtifactWithRepeatedPatch(final String serializedPatch) throws IOException {
final TrieMetadata metadata = new TrieMetadata(6, WordTraversalDirection.BACKWARD,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
final ByteArrayOutputStream rawOutputStream = new ByteArrayOutputStream();
try (DataOutputStream dataOutput = new DataOutputStream(rawOutputStream)) {
dataOutput.writeInt(0x45475452);
dataOutput.writeInt(6);
dataOutput.writeInt(2);
dataOutput.writeInt(0);
dataOutput.writeUTF(metadata.toTextBlock());
dataOutput.writeBoolean(false);
dataOutput.writeInt(1);
dataOutput.writeChar('a');
dataOutput.writeInt(1);
dataOutput.writeInt(1);
dataOutput.writeUTF(serializedPatch);
dataOutput.writeInt(1);
dataOutput.writeBoolean(false);
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeUTF(serializedPatch);
dataOutput.writeInt(2);
}
return gzip(rawOutputStream.toByteArray());
}
/**
* Compresses a binary payload using GZip.
*
* @param payload uncompressed bytes
* @return compressed bytes
* @throws IOException if compression fails
*/
private static byte[] gzip(final byte[] payload) throws IOException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream)) {
gzipOutputStream.write(payload);
}
return outputStream.toByteArray();
}
/**
* Utility method that produces a small GZip-compressed byte array.
*

266
src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java
View File

@@ -34,10 +34,15 @@ import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -58,6 +63,7 @@ import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
import org.egothor.stemmer.trie.CompiledNode;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Tag;
@@ -67,6 +73,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.MockedStatic;
/**
* Professional test suite for {@link StemmerPatchTrieLoader}.
@@ -386,6 +393,161 @@ final class StemmerPatchTrieLoaderTest {
}
}
/**
* Structural tests for direct compiled binary-loader delegation.
*/
@Nested
@DisplayName("Compiled binary routing")
@Tag("unit")
@Tag("io")
@Tag("persistence")
final class CompiledBinaryRoutingTests {
/**
* Verifies that the path overload returns the direct compiled BinaryIO result
* without invoking deprecated String deserialization.
*
* @throws IOException if the loader unexpectedly reports an I/O failure
*/
@Test
@DisplayName("Path compiled loading must use direct compiled BinaryIO")
void shouldRouteCompiledPathLoadingDirectly() throws IOException {
final Path path = tempDir.resolve("compiled-routing-path.bin.gz");
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path)).thenReturn(compiledSentinel);
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path)).thenReturn(stringFallback);
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader.loadBinaryCompiled(path);
assertSame(compiledSentinel, actual,
"Path loading must return the trie materialized by the direct compiled reader.");
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path), times(1));
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path), never());
binaryIo.verifyNoMoreInteractions();
}
}
/**
* Verifies that the path-and-span overload preserves both arguments while
* delegating directly to compiled BinaryIO.
*
* @throws IOException if the loader unexpectedly reports an I/O failure
*/
@Test
@DisplayName("Path and span compiled loading must use direct compiled BinaryIO")
void shouldRouteCompiledPathAndSpanLoadingDirectly() throws IOException {
final Path path = tempDir.resolve("compiled-routing-path-span.bin.gz");
final int maxExpandedIndex = 17;
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex))
.thenReturn(compiledSentinel);
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex)).thenReturn(stringFallback);
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader.loadBinaryCompiled(path,
maxExpandedIndex);
assertSame(compiledSentinel, actual,
"Path-and-span loading must return the trie materialized by the direct compiled reader.");
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex), times(1));
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex), never());
binaryIo.verifyNoMoreInteractions();
}
}
/**
* Verifies that the file-name overload returns the direct compiled BinaryIO
* result without invoking deprecated String deserialization.
*
* @throws IOException if the loader unexpectedly reports an I/O failure
*/
@Test
@DisplayName("String compiled loading must use direct compiled BinaryIO")
void shouldRouteCompiledStringLoadingDirectly() throws IOException {
final String fileName = tempDir.resolve("compiled-routing-string.bin.gz").toString();
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName)).thenReturn(compiledSentinel);
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName)).thenReturn(stringFallback);
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
.loadBinaryCompiled(fileName);
assertSame(compiledSentinel, actual,
"String loading must return the trie materialized by the direct compiled reader.");
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName), times(1));
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName), never());
binaryIo.verifyNoMoreInteractions();
}
}
/**
* Verifies that the file-name-and-span overload preserves both arguments
* while delegating directly to compiled BinaryIO.
*
* @throws IOException if the loader unexpectedly reports an I/O failure
*/
@Test
@DisplayName("String and span compiled loading must use direct compiled BinaryIO")
void shouldRouteCompiledStringAndSpanLoadingDirectly() throws IOException {
final String fileName = tempDir.resolve("compiled-routing-string-span.bin.gz").toString();
final int maxExpandedIndex = 17;
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex))
.thenReturn(compiledSentinel);
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex))
.thenReturn(stringFallback);
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
.loadBinaryCompiled(fileName, maxExpandedIndex);
assertSame(compiledSentinel, actual,
"String-and-span loading must return the trie materialized by the direct compiled reader.");
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex), times(1));
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex), never());
binaryIo.verifyNoMoreInteractions();
}
}
/**
* Verifies that the stream overload preserves stream identity while
* delegating directly to compiled BinaryIO.
*
* @throws IOException if the loader unexpectedly reports an I/O failure
*/
@Test
@DisplayName("InputStream compiled loading must use direct compiled BinaryIO")
void shouldRouteCompiledInputStreamLoadingDirectly() throws IOException {
final InputStream inputStream = new ByteArrayInputStream(new byte[] { 1, 2, 3 });
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream)).thenReturn(compiledSentinel);
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(inputStream)).thenReturn(stringFallback);
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
.loadBinaryCompiled(inputStream);
assertSame(compiledSentinel, actual,
"Stream loading must return the trie materialized by the direct compiled reader.");
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream), times(1));
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(inputStream), never());
binaryIo.verifyNoMoreInteractions();
}
}
}
/**
* Focused internal loader behavior tests.
*/
@@ -707,6 +869,54 @@ final class StemmerPatchTrieLoaderTest {
}
}
/**
* Verifies that public compiled binary loading materializes final command
* arrays directly and preserves shared table identity, counts, metadata, and
* canonical graph size.
*
* @throws IOException if writing or reading fails
*/
@Test
@DisplayName("Compiled binary loading should materialize one final shared command graph")
void shouldMaterializeOneFinalSharedCommandGraphFromBinary() throws IOException {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
final String sharedPatch = encoder.encode("ab", "a");
final String longerDeletionPatch = encoder.encode("xab", "x");
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("ab", new String(sharedPatch), 2);
builder.put("xab", longerDeletionPatch);
builder.put("cb", new String(sharedPatch), 3);
builder.put("ycb", new String(longerDeletionPatch));
final FrequencyTrie<String> sourceTrie = builder.build();
final Path binaryFile = tempDir.resolve("direct-compiled-trie.bin.gz");
StemmerPatchTrieLoader.saveBinary(sourceTrie, binaryFile);
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieLoader
.loadBinaryCompiled(binaryFile);
final FrequencyTrie<String> stringTrie = StemmerPatchTrieLoader.loadBinary(binaryFile);
final CompiledNode<CompiledPatchCommand> suffixBNode = compiledTrie.root().findChild('b');
final CompiledNode<CompiledPatchCommand> abNode = suffixBNode.findChild('a');
final CompiledNode<CompiledPatchCommand> cbNode = suffixBNode.findChild('c');
assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()),
() -> assertEquals(sourceTrie.size(), compiledTrie.size()),
() -> assertEquals(sourceTrie.getEntries("ab").get(0).count(),
compiledTrie.getEntries("ab").get(0).count()),
() -> assertEquals(sourceTrie.getEntries("cb").get(0).count(),
compiledTrie.getEntries("cb").get(0).count()),
() -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()),
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
() -> assertEquals("a", compiledTrie.get("ab").apply("ab")),
() -> assertEquals("c", compiledTrie.get("cb").apply("cb")),
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")),
() -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")),
() -> assertInstanceOf(String.class, stringTrie.get("ab")),
() -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab")));
}
/**
* Verifies that binary load overloads with an explicit dense lookup span
* preserve trie semantics while honoring the dense-layout override.
@@ -728,9 +938,17 @@ final class StemmerPatchTrieLoaderTest {
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX);
-1);
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
final FrequencyTrie<CompiledPatchCommand> compiledDefault = StemmerPatchTrieLoader
.loadBinaryCompiled(binaryFile);
final FrequencyTrie<CompiledPatchCommand> compiledDefaultByNegative = StemmerPatchTrieLoader
.loadBinaryCompiled(binaryFile, -1);
final FrequencyTrie<CompiledPatchCommand> compiledNoDense = StemmerPatchTrieLoader
.loadBinaryCompiled(binaryFile, 0);
final FrequencyTrie<CompiledPatchCommand> compiledStringNoDense = StemmerPatchTrieLoader
.loadBinaryCompiled(binaryFile.toString(), 0);
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
@@ -738,11 +956,27 @@ final class StemmerPatchTrieLoaderTest {
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
"studying");
assertCompiledTrieSemanticsEqual(original, compiledDefault, "run", "running", "runner", "cities",
"studying");
assertCompiledTrieSemanticsEqual(original, compiledDefaultByNegative, "run", "running", "runner",
"cities", "studying");
assertCompiledTrieSemanticsEqual(original, compiledNoDense, "run", "running", "runner", "cities",
"studying");
assertCompiledTrieSemanticsEqual(original, compiledStringNoDense, "run", "running", "runner", "cities",
"studying");
assertTrue(compiledDefault.root().hasDenseLookup(),
"Default compiled loading should use the documented dense lookup span.");
assertTrue(compiledDefaultByNegative.root().hasDenseLookup(),
"Negative compiled override should use the documented default dense lookup span.");
assertFalse(fromPathNoDense.root().hasDenseLookup(),
"Zero span should disable dense lookup on the loaded root.");
assertFalse(fromStringNoDense.root().hasDenseLookup(),
"Zero span should disable dense lookup on the loaded root.");
assertFalse(compiledNoDense.root().hasDenseLookup(),
"Zero span should disable dense lookup on the directly compiled root.");
assertFalse(compiledStringNoDense.root().hasDenseLookup(),
"String-path zero span should disable dense lookup on the directly compiled root.");
}
/**
@@ -990,6 +1224,36 @@ final class StemmerPatchTrieLoaderTest {
return stems;
}
/**
* Creates a compiled trie whose identity distinguishes the direct BinaryIO
* result from any graph produced through deprecated String loading.
*
* @return compiled trie sentinel
*/
private static FrequencyTrie<CompiledPatchCommand> compiledRoutingSentinel() {
final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run");
final CompiledPatchCommand compiledPatch = CompiledPatchCommand.compile(serializedPatch,
WordTraversalDirection.BACKWARD);
return new FrequencyTrie.Builder<CompiledPatchCommand>(CompiledPatchCommand[]::new,
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE))
.put("running", compiledPatch)
.build();
}
/**
* Creates a valid String-valued trie that allows the deprecated two-stage route
* to execute rather than failing because of incomplete mock setup.
*
* @return valid String-valued fallback trie
*/
private static FrequencyTrie<String> stringRoutingFallback() {
final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run");
return new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE))
.put("running", serializedPatch)
.build();
}
/**
* Verifies semantic equality of two tries for the supplied words by comparing
* both their raw patch arrays and reconstructed stem sets.