perf(loader): materialize compiled patch tries directly
This commit is contained in:
@@ -242,6 +242,18 @@ public final class FrequencyTrie<V> {
|
|||||||
return STREAM_VERSION;
|
return STREAM_VERSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns whether the supplied metadata identifies a stream format with a
|
||||||
|
* serialized value table.
|
||||||
|
*
|
||||||
|
* @param metadata parsed trie metadata
|
||||||
|
* @return {@code true} when values are stored in a stream-local table
|
||||||
|
* @throws NullPointerException if {@code metadata} is {@code null}
|
||||||
|
*/
|
||||||
|
/* default */ static boolean usesValueTableFormat(final TrieMetadata metadata) {
|
||||||
|
return Objects.requireNonNull(metadata, "metadata").formatVersion() >= VALUE_TABLE_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Receives trie values during visitor-style lookup.
|
* Receives trie values during visitor-style lookup.
|
||||||
*
|
*
|
||||||
@@ -817,7 +829,37 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||||
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
||||||
return CompiledTrieReader.read(inputStream, arrayFactory, valueCodec, maxExpandedIndex);
|
Objects.requireNonNull(valueCodec, "valueCodec");
|
||||||
|
return readFromWithMetadata(inputStream, arrayFactory,
|
||||||
|
(dataInput, metadata) -> valueCodec.read(dataInput), maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compiled trie while allowing value decoding to use already parsed
|
||||||
|
* trie metadata.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* This package-private path materializes the requested final value type during
|
||||||
|
* the normal graph read. It does not expose reader state or construct an
|
||||||
|
* intermediate trie with a different value type.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param inputStream source input stream
|
||||||
|
* @param arrayFactory factory used to create typed value arrays
|
||||||
|
* @param valueReader metadata-aware value reader
|
||||||
|
* @param maxExpandedIndex dense lookup span override; zero disables dense
|
||||||
|
* lookup, negative values use
|
||||||
|
* {@link #DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @param <V> final value type
|
||||||
|
* @return deserialized compiled trie containing values returned by
|
||||||
|
* {@code valueReader}
|
||||||
|
* @throws NullPointerException if any argument is {@code null}
|
||||||
|
* @throws IOException if reading fails or the binary format is invalid
|
||||||
|
*/
|
||||||
|
/* default */ static <V> FrequencyTrie<V> readFromWithMetadata(final InputStream inputStream,
|
||||||
|
final IntFunction<V[]> arrayFactory, final MetadataValueStreamReader<V> valueReader,
|
||||||
|
final int maxExpandedIndex) throws IOException {
|
||||||
|
return CompiledTrieReader.read(inputStream, arrayFactory, valueReader, maxExpandedIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1044,10 +1086,10 @@ public final class FrequencyTrie<V> {
|
|||||||
private static final class CompiledTrieReader {
|
private static final class CompiledTrieReader {
|
||||||
|
|
||||||
private static <V> FrequencyTrie<V> read(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
private static <V> FrequencyTrie<V> read(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||||
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
final MetadataValueStreamReader<V> valueReader, final int maxExpandedIndex) throws IOException {
|
||||||
Objects.requireNonNull(inputStream, "inputStream");
|
Objects.requireNonNull(inputStream, "inputStream");
|
||||||
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||||
Objects.requireNonNull(valueCodec, "valueCodec");
|
Objects.requireNonNull(valueReader, "valueReader");
|
||||||
if (maxExpandedIndex < -1) {
|
if (maxExpandedIndex < -1) {
|
||||||
throw new IllegalArgumentException("maxExpandedIndex must be >= -1.");
|
throw new IllegalArgumentException("maxExpandedIndex must be >= -1.");
|
||||||
}
|
}
|
||||||
@@ -1074,11 +1116,12 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
|
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
|
||||||
final V[] valueTable = version >= VALUE_TABLE_VERSION ? readValueTable(dataInput, arrayFactory, valueCodec)
|
final V[] valueTable = version >= VALUE_TABLE_VERSION
|
||||||
|
? readValueTable(dataInput, arrayFactory, valueReader, sourceMetadata)
|
||||||
: null;
|
: null;
|
||||||
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
|
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
|
||||||
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, valueTable, nodeCount,
|
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueReader, sourceMetadata, valueTable,
|
||||||
effectiveMaxExpandedIndex, version);
|
nodeCount, effectiveMaxExpandedIndex, version);
|
||||||
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
||||||
|
|
||||||
if (LOGGER.isLoggable(Level.FINE)) {
|
if (LOGGER.isLoggable(Level.FINE)) {
|
||||||
@@ -1099,13 +1142,14 @@ public final class FrequencyTrie<V> {
|
|||||||
*
|
*
|
||||||
* @param dataInput input stream
|
* @param dataInput input stream
|
||||||
* @param arrayFactory typed-array factory
|
* @param arrayFactory typed-array factory
|
||||||
* @param valueCodec codec responsible for value decoding
|
* @param valueReader metadata-aware value reader
|
||||||
|
* @param metadata parsed trie metadata
|
||||||
* @param <V> value type
|
* @param <V> value type
|
||||||
* @return decoded values in table-index order
|
* @return decoded values in table-index order
|
||||||
* @throws IOException if the count is negative or value decoding fails
|
* @throws IOException if the count is negative or value decoding fails
|
||||||
*/
|
*/
|
||||||
private static <V> V[] readValueTable(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
|
private static <V> V[] readValueTable(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
|
||||||
final ValueStreamCodec<V> valueCodec) throws IOException {
|
final MetadataValueStreamReader<V> valueReader, final TrieMetadata metadata) throws IOException {
|
||||||
final int distinctValueCount = dataInput.readInt();
|
final int distinctValueCount = dataInput.readInt();
|
||||||
if (distinctValueCount < 0) {
|
if (distinctValueCount < 0) {
|
||||||
throw new IOException("Negative distinct value count: " + distinctValueCount);
|
throw new IOException("Negative distinct value count: " + distinctValueCount);
|
||||||
@@ -1113,7 +1157,7 @@ public final class FrequencyTrie<V> {
|
|||||||
|
|
||||||
final V[] valueTable = arrayFactory.apply(distinctValueCount);
|
final V[] valueTable = arrayFactory.apply(distinctValueCount);
|
||||||
for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) {
|
for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) {
|
||||||
valueTable[valueIndex] = valueCodec.read(dataInput);
|
valueTable[valueIndex] = valueReader.read(dataInput, metadata);
|
||||||
}
|
}
|
||||||
return valueTable;
|
return valueTable;
|
||||||
}
|
}
|
||||||
@@ -1181,8 +1225,9 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
|
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
|
||||||
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final V[] valueTable,
|
final IntFunction<V[]> arrayFactory, final MetadataValueStreamReader<V> valueReader,
|
||||||
final int nodeCount, final int maxExpandedIndex, final int version) throws IOException {
|
final TrieMetadata metadata, final V[] valueTable, final int nodeCount, final int maxExpandedIndex,
|
||||||
|
final int version) throws IOException {
|
||||||
final char[][] edgeLabelsByNode = new char[nodeCount][];
|
final char[][] edgeLabelsByNode = new char[nodeCount][];
|
||||||
final int[][] childNodeIdsByNode = new int[nodeCount][];
|
final int[][] childNodeIdsByNode = new int[nodeCount][];
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
@@ -1234,7 +1279,7 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex];
|
orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex];
|
||||||
} else {
|
} else {
|
||||||
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
|
orderedValuesByNode[nodeIndex][valueIndex] = valueReader.read(dataInput, metadata);
|
||||||
}
|
}
|
||||||
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
|
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
|
||||||
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
|
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
|
||||||
@@ -1980,6 +2025,32 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads one final trie value using metadata parsed from the same binary stream.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Implementations are invoked only during deserialization and are not retained
|
||||||
|
* by the resulting trie. Returned values are stored directly in final compiled
|
||||||
|
* node arrays.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param <V> final value type
|
||||||
|
*/
|
||||||
|
/* default */
|
||||||
|
@FunctionalInterface
|
||||||
|
interface MetadataValueStreamReader<V> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads and materializes one final value.
|
||||||
|
*
|
||||||
|
* @param dataInput source data input
|
||||||
|
* @param metadata already parsed trie metadata
|
||||||
|
* @return final value to store directly in compiled nodes
|
||||||
|
* @throws IOException if reading or materialization fails
|
||||||
|
*/
|
||||||
|
V read(DataInputStream dataInput, TrieMetadata metadata) throws IOException;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Codec used to persist values stored in the trie.
|
* Codec used to persist values stored in the trie.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -39,7 +39,10 @@ import java.io.InputStream;
|
|||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.function.BiFunction;
|
||||||
import java.util.logging.Level;
|
import java.util.logging.Level;
|
||||||
import java.util.logging.Logger;
|
import java.util.logging.Logger;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
@@ -53,6 +56,8 @@ import java.util.zip.GZIPOutputStream;
|
|||||||
* patch commands represented as {@link String}. The serialized trie payload is
|
* patch commands represented as {@link String}. The serialized trie payload is
|
||||||
* the native binary format of {@link FrequencyTrie}, wrapped in GZip
|
* the native binary format of {@link FrequencyTrie}, wrapped in GZip
|
||||||
* compression.
|
* compression.
|
||||||
|
* Binary reads can either preserve those serialized strings or materialize
|
||||||
|
* {@link CompiledPatchCommand} values directly in the final trie nodes.
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* The helper centralizes the codec and compression details so that higher-level
|
* The helper centralizes the codec and compression details so that higher-level
|
||||||
@@ -71,6 +76,21 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
*/
|
*/
|
||||||
private static final FrequencyTrie.ValueStreamCodec<String> STRING_CODEC = new StringValueStreamCodec();
|
private static final FrequencyTrie.ValueStreamCodec<String> STRING_CODEC = new StringValueStreamCodec();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum serialized patch-command length included in validation diagnostics.
|
||||||
|
*/
|
||||||
|
private static final int MAX_DIAGNOSTIC_PATCH_LENGTH = 128;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Null-check parameter name for filesystem paths.
|
||||||
|
*/
|
||||||
|
private static final String PATH_PARAMETER = "path";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Null-check parameter name for filesystem path strings.
|
||||||
|
*/
|
||||||
|
private static final String FILE_NAME_PARAMETER = "fileName";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility class.
|
* Utility class.
|
||||||
*/
|
*/
|
||||||
@@ -87,7 +107,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> read(final Path path) throws IOException {
|
public static FrequencyTrie<String> read(final Path path) throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
|
|
||||||
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||||
return read(fileInputStream);
|
return read(fileInputStream);
|
||||||
@@ -110,7 +130,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
|
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
|
|
||||||
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||||
return read(fileInputStream, maxExpandedIndex);
|
return read(fileInputStream, maxExpandedIndex);
|
||||||
@@ -127,7 +147,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> read(final String fileName) throws IOException {
|
public static FrequencyTrie<String> read(final String fileName) throws IOException {
|
||||||
Objects.requireNonNull(fileName, "fileName");
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
return read(Path.of(fileName));
|
return read(Path.of(fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,7 +167,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
|
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
|
||||||
Objects.requireNonNull(fileName, "fileName");
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
return read(Path.of(fileName), maxExpandedIndex);
|
return read(Path.of(fileName), maxExpandedIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -204,6 +224,149 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* a filesystem path.
|
||||||
|
*
|
||||||
|
* @param path source file
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code path} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final Path path) throws IOException {
|
||||||
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
|
|
||||||
|
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||||
|
return readCompiled(fileInputStream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* a filesystem path with a dense child lookup span override.
|
||||||
|
*
|
||||||
|
* @param path source file
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code path} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final Path path,
|
||||||
|
final int maxExpandedIndex)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
|
|
||||||
|
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||||
|
return readCompiled(fileInputStream, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* a filesystem path string.
|
||||||
|
*
|
||||||
|
* @param fileName source file name or path string
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final String fileName) throws IOException {
|
||||||
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
|
return readCompiled(Path.of(fileName));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* a filesystem path string with a dense child lookup span override.
|
||||||
|
*
|
||||||
|
* @param fileName source file name or path string
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final String fileName,
|
||||||
|
final int maxExpandedIndex)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
|
return readCompiled(Path.of(fileName), maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* an input stream.
|
||||||
|
*
|
||||||
|
* @param inputStream source stream
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream)
|
||||||
|
throws IOException {
|
||||||
|
return readCompiled(inputStream, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie directly as compiled values from
|
||||||
|
* an input stream with a dense child lookup span override.
|
||||||
|
*
|
||||||
|
* @param inputStream source stream
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream,
|
||||||
|
final int maxExpandedIndex) throws IOException {
|
||||||
|
return readCompiled(inputStream, maxExpandedIndex, CompiledPatchCommand::compile);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a compressed binary patch-command trie using a caller-supplied command
|
||||||
|
* compiler.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* This package-private seam permits deterministic compilation-count testing
|
||||||
|
* without global counters. Production callers use
|
||||||
|
* {@link CompiledPatchCommand#compile(String, WordTraversalDirection)}.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param inputStream source stream
|
||||||
|
* @param maxExpandedIndex dense lookup span override
|
||||||
|
* @param commandCompiler compiler for one serialized command and traversal
|
||||||
|
* direction
|
||||||
|
* @return directly materialized compiled patch-command trie
|
||||||
|
* @throws NullPointerException if any argument is {@code null}
|
||||||
|
* @throws IOException if reading, decompression, or command compilation
|
||||||
|
* fails
|
||||||
|
*/
|
||||||
|
/* default */ static FrequencyTrie<CompiledPatchCommand> readCompiled(final InputStream inputStream,
|
||||||
|
final int maxExpandedIndex,
|
||||||
|
final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(inputStream, "inputStream");
|
||||||
|
Objects.requireNonNull(commandCompiler, "commandCompiler");
|
||||||
|
final CompiledPatchValueReader valueReader = new CompiledPatchValueReader(commandCompiler);
|
||||||
|
|
||||||
|
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));
|
||||||
|
DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) {
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> trie = FrequencyTrie.readFromWithMetadata(dataInputStream,
|
||||||
|
CompiledPatchCommand[]::new, valueReader, maxExpandedIndex);
|
||||||
|
|
||||||
|
LOGGER.log(Level.FINE, "Read compressed binary stemmer trie directly as compiled patch commands.");
|
||||||
|
return trie;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||||
* at a filesystem path.
|
* at a filesystem path.
|
||||||
@@ -214,7 +377,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static TrieMetadata readMetadata(final Path path) throws IOException {
|
public static TrieMetadata readMetadata(final Path path) throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
return read(path).metadata();
|
return read(path).metadata();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,7 +391,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if reading or decompression fails
|
* @throws IOException if reading or decompression fails
|
||||||
*/
|
*/
|
||||||
public static TrieMetadata readMetadata(final String fileName) throws IOException {
|
public static TrieMetadata readMetadata(final String fileName) throws IOException {
|
||||||
Objects.requireNonNull(fileName, "fileName");
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
return readMetadata(Path.of(fileName));
|
return readMetadata(Path.of(fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,7 +419,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
*/
|
*/
|
||||||
public static void write(final FrequencyTrie<String> trie, final Path path) throws IOException {
|
public static void write(final FrequencyTrie<String> trie, final Path path) throws IOException {
|
||||||
Objects.requireNonNull(trie, "trie");
|
Objects.requireNonNull(trie, "trie");
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PATH_PARAMETER);
|
||||||
|
|
||||||
final Path parent = path.toAbsolutePath().getParent();
|
final Path parent = path.toAbsolutePath().getParent();
|
||||||
if (parent != null) {
|
if (parent != null) {
|
||||||
@@ -278,7 +441,7 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
* @throws IOException if writing fails
|
* @throws IOException if writing fails
|
||||||
*/
|
*/
|
||||||
public static void write(final FrequencyTrie<String> trie, final String fileName) throws IOException {
|
public static void write(final FrequencyTrie<String> trie, final String fileName) throws IOException {
|
||||||
Objects.requireNonNull(fileName, "fileName");
|
Objects.requireNonNull(fileName, FILE_NAME_PARAMETER);
|
||||||
write(trie, Path.of(fileName));
|
write(trie, Path.of(fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -323,4 +486,97 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
return dataInput.readUTF();
|
return dataInput.readUTF();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Metadata-aware reader that compiles serialized patch commands into final trie
|
||||||
|
* values.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Version 7 value tables already contain distinct serialized values, so each
|
||||||
|
* entry is compiled directly. Historical inline formats use the reader-local
|
||||||
|
* equality cache to compile repeated serialized commands once. Neither the cache
|
||||||
|
* nor this reader is retained after trie loading.
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
private static final class CompiledPatchValueReader
|
||||||
|
implements FrequencyTrie.MetadataValueStreamReader<CompiledPatchCommand> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compiler used to materialize one final command.
|
||||||
|
*/
|
||||||
|
private final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compatibility cache used only by historical inline-value streams.
|
||||||
|
*/
|
||||||
|
private final Map<String, CompiledPatchCommand> legacyCompiledCommands = new HashMap<>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates one reader with a caller-supplied compiler.
|
||||||
|
*
|
||||||
|
* @param commandCompiler compiler for serialized patch commands
|
||||||
|
*/
|
||||||
|
private CompiledPatchValueReader(
|
||||||
|
final BiFunction<String, WordTraversalDirection, CompiledPatchCommand> commandCompiler) {
|
||||||
|
this.commandCompiler = commandCompiler;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads and compiles one serialized patch command.
|
||||||
|
*
|
||||||
|
* @param dataInput source data input
|
||||||
|
* @param metadata parsed trie metadata
|
||||||
|
* @return final compiled patch command
|
||||||
|
* @throws IOException if reading or command compilation fails
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public CompiledPatchCommand read(final DataInputStream dataInput, final TrieMetadata metadata)
|
||||||
|
throws IOException {
|
||||||
|
final String serializedPatch = dataInput.readUTF();
|
||||||
|
if (FrequencyTrie.usesValueTableFormat(metadata)) {
|
||||||
|
return compile(serializedPatch, metadata.traversalDirection());
|
||||||
|
}
|
||||||
|
|
||||||
|
final CompiledPatchCommand cachedCommand = this.legacyCompiledCommands.get(serializedPatch);
|
||||||
|
if (cachedCommand != null) {
|
||||||
|
return cachedCommand;
|
||||||
|
}
|
||||||
|
final CompiledPatchCommand compiledCommand = compile(serializedPatch, metadata.traversalDirection());
|
||||||
|
this.legacyCompiledCommands.put(serializedPatch, compiledCommand);
|
||||||
|
return compiledCommand;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compiles one serialized command and converts validation failures to
|
||||||
|
* trust-boundary {@link IOException} instances.
|
||||||
|
*
|
||||||
|
* @param serializedPatch serialized patch command
|
||||||
|
* @param traversalDirection traversal direction from persisted metadata
|
||||||
|
* @return compiled patch command
|
||||||
|
* @throws IOException if the serialized command is invalid
|
||||||
|
*/
|
||||||
|
private CompiledPatchCommand compile(final String serializedPatch,
|
||||||
|
final WordTraversalDirection traversalDirection) throws IOException {
|
||||||
|
try {
|
||||||
|
return this.commandCompiler.apply(serializedPatch, traversalDirection);
|
||||||
|
} catch (IllegalArgumentException exception) {
|
||||||
|
throw new IOException("Invalid persisted patch command '" + boundedPatch(serializedPatch)
|
||||||
|
+ "' for traversal direction " + traversalDirection + '.', exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a safely bounded patch-command representation for diagnostics.
|
||||||
|
*
|
||||||
|
* @param serializedPatch serialized patch command
|
||||||
|
* @return complete or length-bounded diagnostic representation
|
||||||
|
*/
|
||||||
|
private static String boundedPatch(final String serializedPatch) {
|
||||||
|
if (serializedPatch.length() <= MAX_DIAGNOSTIC_PATCH_LENGTH) {
|
||||||
|
return serializedPatch;
|
||||||
|
}
|
||||||
|
return serializedPatch.substring(0, MAX_DIAGNOSTIC_PATCH_LENGTH) + "... (length "
|
||||||
|
+ serializedPatch.length() + ')';
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1311,6 +1311,12 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path and
|
* Loads a GZip-compressed binary patch-command trie from a filesystem path and
|
||||||
* returns runtime-specialized compiled patch values.
|
* returns runtime-specialized compiled patch values.
|
||||||
*
|
*
|
||||||
|
* <p>
|
||||||
|
* Serialized patch commands are compiled while the binary graph is read, so the
|
||||||
|
* returned trie is materialized directly without an intermediate
|
||||||
|
* {@code FrequencyTrie<String>} or a second node graph.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
* @param path path to the compressed binary trie file
|
* @param path path to the compressed binary trie file
|
||||||
* @return compiled patch-command trie with runtime-specialized values
|
* @return compiled patch-command trie with runtime-specialized values
|
||||||
* @throws NullPointerException if {@code path} is {@code null}
|
* @throws NullPointerException if {@code path} is {@code null}
|
||||||
@@ -1318,7 +1324,8 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* read
|
* read
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path) throws IOException {
|
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path) throws IOException {
|
||||||
return compilePatchTrie(loadBinary(path));
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
|
return StemmerPatchTrieBinaryIO.readCompiled(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1351,6 +1358,11 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* a custom dense lookup span override and returns runtime-specialized compiled
|
* a custom dense lookup span override and returns runtime-specialized compiled
|
||||||
* patch values.
|
* patch values.
|
||||||
*
|
*
|
||||||
|
* <p>
|
||||||
|
* Serialized patch commands are compiled directly into the final node graph.
|
||||||
|
* The dense lookup override is applied during that graph materialization.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
* @param path path to the compressed binary trie file
|
* @param path path to the compressed binary trie file
|
||||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
@@ -1361,7 +1373,8 @@ public final class StemmerPatchTrieLoader {
|
|||||||
*/
|
*/
|
||||||
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path,
|
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final Path path,
|
||||||
final int maxExpandedIndex) throws IOException {
|
final int maxExpandedIndex) throws IOException {
|
||||||
return compilePatchTrie(loadBinary(path, maxExpandedIndex));
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
|
return StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1387,6 +1400,11 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
* string and returns runtime-specialized compiled patch values.
|
* string and returns runtime-specialized compiled patch values.
|
||||||
*
|
*
|
||||||
|
* <p>
|
||||||
|
* Serialized patch commands are compiled while the binary graph is read, so no
|
||||||
|
* intermediate String-valued trie is constructed.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
* @param fileName file name or path string
|
* @param fileName file name or path string
|
||||||
* @return compiled patch-command trie with runtime-specialized values
|
* @return compiled patch-command trie with runtime-specialized values
|
||||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||||
@@ -1394,7 +1412,8 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* read
|
* read
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName) throws IOException {
|
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName) throws IOException {
|
||||||
return compilePatchTrie(loadBinary(fileName));
|
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||||
|
return StemmerPatchTrieBinaryIO.readCompiled(fileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1428,6 +1447,11 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* using a custom dense lookup span override and returns runtime-specialized
|
* using a custom dense lookup span override and returns runtime-specialized
|
||||||
* compiled patch values.
|
* compiled patch values.
|
||||||
*
|
*
|
||||||
|
* <p>
|
||||||
|
* Serialized patch commands are compiled directly into the final node graph.
|
||||||
|
* The dense lookup override is applied during that graph materialization.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
* @param fileName file name or path string
|
* @param fileName file name or path string
|
||||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
@@ -1438,7 +1462,8 @@ public final class StemmerPatchTrieLoader {
|
|||||||
*/
|
*/
|
||||||
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName,
|
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final String fileName,
|
||||||
final int maxExpandedIndex) throws IOException {
|
final int maxExpandedIndex) throws IOException {
|
||||||
return compilePatchTrie(loadBinary(fileName, maxExpandedIndex));
|
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||||
|
return StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1462,6 +1487,12 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* Loads a GZip-compressed binary patch-command trie from an input stream and
|
* Loads a GZip-compressed binary patch-command trie from an input stream and
|
||||||
* returns runtime-specialized compiled patch values.
|
* returns runtime-specialized compiled patch values.
|
||||||
*
|
*
|
||||||
|
* <p>
|
||||||
|
* Serialized patch commands are compiled while the binary graph is read, so the
|
||||||
|
* returned trie is materialized directly without an intermediate
|
||||||
|
* {@code FrequencyTrie<String>} or graph-mapping pass.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
* @param inputStream source input stream
|
* @param inputStream source input stream
|
||||||
* @return compiled patch-command trie with runtime-specialized values
|
* @return compiled patch-command trie with runtime-specialized values
|
||||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||||
@@ -1469,7 +1500,8 @@ public final class StemmerPatchTrieLoader {
|
|||||||
*/
|
*/
|
||||||
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final InputStream inputStream)
|
public static FrequencyTrie<CompiledPatchCommand> loadBinaryCompiled(final InputStream inputStream)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return compilePatchTrie(loadBinary(inputStream));
|
Objects.requireNonNull(inputStream, "inputStream");
|
||||||
|
return StemmerPatchTrieBinaryIO.readCompiled(inputStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -150,6 +150,25 @@ final class CompiledTrieArtifactRegressionTest {
|
|||||||
() -> assertGoldenArtifactSemanticProbes(trie, artifactCase));
|
() -> assertGoldenArtifactSemanticProbes(trie, artifactCase));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that every committed version 6 artifact can be materialized directly
|
||||||
|
* as compiled commands while preserving representative stemming behavior.
|
||||||
|
*
|
||||||
|
* @param artifactCase regression case
|
||||||
|
* @throws IOException if artifact loading fails
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "{0}")
|
||||||
|
@MethodSource("artifactCases")
|
||||||
|
@DisplayName("Committed golden artifacts must load directly as compiled commands")
|
||||||
|
void shouldLoadGoldenArtifactsDirectlyAsCompiledCommands(final ArtifactCase artifactCase) throws IOException {
|
||||||
|
final byte[] goldenArtifactBytes = RegressionArtifactSupport
|
||||||
|
.readResourceBytes(artifactCase.goldenArtifactResource());
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> trie = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(new ByteArrayInputStream(goldenArtifactBytes));
|
||||||
|
|
||||||
|
assertGoldenCompiledArtifactSemanticProbes(trie, artifactCase);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies in-process determinism independently of the checked-in golden file
|
* Verifies in-process determinism independently of the checked-in golden file
|
||||||
* by compiling the same dictionary twice and requiring identical artifact
|
* by compiling the same dictionary twice and requiring identical artifact
|
||||||
@@ -264,6 +283,35 @@ final class CompiledTrieArtifactRegressionTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies representative semantic probes against a directly materialized
|
||||||
|
* compiled-command trie.
|
||||||
|
*
|
||||||
|
* @param trie compiled-command trie to inspect
|
||||||
|
* @param artifactCase regression case providing the expected probes
|
||||||
|
*/
|
||||||
|
private static void assertGoldenCompiledArtifactSemanticProbes(
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> trie, final ArtifactCase artifactCase) {
|
||||||
|
for (ProbeExpectation probe : artifactCase.probes()) {
|
||||||
|
final CompiledPatchCommand[] allPatchCommands = trie.getAll(probe.word());
|
||||||
|
final CompiledPatchCommand preferredPatchCommand = trie.get(probe.word());
|
||||||
|
final String preferredStem = preferredPatchCommand == null ? null : preferredPatchCommand.apply(probe.word());
|
||||||
|
final Set<String> allStems = new LinkedHashSet<String>();
|
||||||
|
for (CompiledPatchCommand patchCommand : allPatchCommands) {
|
||||||
|
allStems.add(patchCommand.apply(probe.word()));
|
||||||
|
}
|
||||||
|
|
||||||
|
assertAll(
|
||||||
|
() -> assertFalse(allPatchCommands.length == 0,
|
||||||
|
"Representative probe must produce at least one result for word: " + probe.word()),
|
||||||
|
() -> assertEquals(probe.preferredStem(), preferredStem,
|
||||||
|
"Preferred stem mismatch for representative probe word: " + probe.word()),
|
||||||
|
() -> assertTrue(allStems.containsAll(probe.acceptableStems()),
|
||||||
|
"All acceptable stems must be present in getAll() for representative probe word: "
|
||||||
|
+ probe.word()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Immutable regression case definition.
|
* Immutable regression case definition.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -1112,6 +1112,82 @@ class FrequencyTrieTest {
|
|||||||
() -> assertNull(restored.get("missing")));
|
() -> assertNull(restored.get("missing")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that metadata-aware version 7 reads receive parsed metadata before
|
||||||
|
* decoding and materialize shared final values directly in compiled nodes.
|
||||||
|
*
|
||||||
|
* @throws IOException if test I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("Metadata-aware version 7 reader materializes shared final values")
|
||||||
|
void metadataAwareVersionSevenReaderMaterializesSharedFinalValues() throws IOException {
|
||||||
|
final FrequencyTrie<String> original = sharedValueTrie();
|
||||||
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||||
|
original.writeTo(outputStream, STRING_CODEC);
|
||||||
|
final AtomicInteger readCount = new AtomicInteger();
|
||||||
|
final AtomicInteger observedMetadataVersion = new AtomicInteger();
|
||||||
|
|
||||||
|
final FrequencyTrie<StringBuilder> restored = FrequencyTrie.readFromWithMetadata(
|
||||||
|
new ByteArrayInputStream(outputStream.toByteArray()), StringBuilder[]::new,
|
||||||
|
(dataInput, metadata) -> {
|
||||||
|
observedMetadataVersion.set(metadata.formatVersion());
|
||||||
|
readCount.incrementAndGet();
|
||||||
|
return new StringBuilder(dataInput.readUTF());
|
||||||
|
}, -1);
|
||||||
|
final CompiledNode<StringBuilder> suffixBNode = restored.root().findChild('b');
|
||||||
|
final CompiledNode<StringBuilder> abNode = suffixBNode.findChild('a');
|
||||||
|
final CompiledNode<StringBuilder> cbNode = suffixBNode.findChild('c');
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(7, observedMetadataVersion.get()),
|
||||||
|
() -> assertEquals(3, readCount.get()),
|
||||||
|
() -> assertSame(restored.get("ab"), restored.get("cb")),
|
||||||
|
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
|
||||||
|
() -> assertEquals(StringBuilder[].class, abNode.orderedValues().getClass()),
|
||||||
|
() -> assertEquals("left", restored.get("xab").toString()),
|
||||||
|
() -> assertEquals("right", restored.get("ycb").toString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that metadata-aware reading preserves the inline value layout used
|
||||||
|
* by every historical stream version from 1 through 6.
|
||||||
|
*
|
||||||
|
* @throws IOException if test I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("Metadata-aware reader supports inline values in versions 1 through 6")
|
||||||
|
void metadataAwareReaderSupportsInlineValuesInVersionsOneThroughSix() throws IOException {
|
||||||
|
for (int version = 1; version <= 6; version++) {
|
||||||
|
final int historicalVersion = version;
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, historicalVersion, 1, 0,
|
||||||
|
dataOutput -> writeMetadataForHistoricalVersion(dataOutput, historicalVersion),
|
||||||
|
new NodeWriter[] { dataOutput -> {
|
||||||
|
if (historicalVersion >= 6) {
|
||||||
|
dataOutput.writeBoolean(false);
|
||||||
|
}
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeUTF("inline-" + historicalVersion);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
} });
|
||||||
|
final AtomicInteger readCount = new AtomicInteger();
|
||||||
|
final AtomicInteger observedMetadataVersion = new AtomicInteger();
|
||||||
|
|
||||||
|
final FrequencyTrie<StringBuilder> trie = FrequencyTrie.readFromWithMetadata(
|
||||||
|
new ByteArrayInputStream(bytes), StringBuilder[]::new,
|
||||||
|
(dataInput, metadata) -> {
|
||||||
|
observedMetadataVersion.set(metadata.formatVersion());
|
||||||
|
readCount.incrementAndGet();
|
||||||
|
return new StringBuilder(dataInput.readUTF());
|
||||||
|
}, -1);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(historicalVersion, observedMetadataVersion.get()),
|
||||||
|
() -> assertEquals(1, readCount.get()),
|
||||||
|
() -> assertEquals("inline-" + historicalVersion, trie.get("").toString()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies fingerprint stability and sensitivity to metadata and trie content.
|
* Verifies fingerprint stability and sensitivity to metadata and trie content.
|
||||||
*/
|
*/
|
||||||
@@ -2325,6 +2401,42 @@ class FrequencyTrieTest {
|
|||||||
dataOutput.writeInt(occurrenceCount);
|
dataOutput.writeInt(occurrenceCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes the metadata layout used by one historical stream version.
|
||||||
|
*
|
||||||
|
* @param dataOutput output stream
|
||||||
|
* @param version historical stream version from 1 through 6
|
||||||
|
* @throws IOException if writing fails
|
||||||
|
*/
|
||||||
|
private static void writeMetadataForHistoricalVersion(final DataOutputStream dataOutput, final int version)
|
||||||
|
throws IOException {
|
||||||
|
if (version == 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (version == 2) {
|
||||||
|
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final ReductionSettings reductionSettings = ReductionSettings
|
||||||
|
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||||
|
if (version <= 4) {
|
||||||
|
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
|
||||||
|
dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal());
|
||||||
|
if (version == 4) {
|
||||||
|
dataOutput.writeInt(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT.ordinal());
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final TrieMetadata metadata = new TrieMetadata(version, WordTraversalDirection.BACKWARD, reductionSettings,
|
||||||
|
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||||
|
dataOutput.writeUTF(metadata.toTextBlock());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes one synthetic metadata block.
|
* Writes one synthetic metadata block.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ package org.egothor.stemmer;
|
|||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||||
@@ -50,14 +51,18 @@ import java.io.ByteArrayOutputStream;
|
|||||||
import java.io.DataInputStream;
|
import java.io.DataInputStream;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.lang.reflect.Constructor;
|
import java.lang.reflect.Constructor;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
|
import org.egothor.stemmer.trie.CompiledNode;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.Nested;
|
import org.junit.jupiter.api.Nested;
|
||||||
import org.junit.jupiter.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
@@ -98,6 +103,17 @@ import org.mockito.MockedStatic;
|
|||||||
@DisplayName("StemmerPatchTrieBinaryIO")
|
@DisplayName("StemmerPatchTrieBinaryIO")
|
||||||
class StemmerPatchTrieBinaryIOTest {
|
class StemmerPatchTrieBinaryIOTest {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum invalid patch-command length retained verbatim in production
|
||||||
|
* diagnostics.
|
||||||
|
*/
|
||||||
|
private static final int DIAGNOSTIC_PATCH_BOUNDARY = 128;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Marker that introduces the original length after diagnostic truncation.
|
||||||
|
*/
|
||||||
|
private static final String DIAGNOSTIC_TRUNCATION_MARKER = "... (length ";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Temporary directory provided by JUnit.
|
* Temporary directory provided by JUnit.
|
||||||
*/
|
*/
|
||||||
@@ -323,7 +339,202 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
"read(InputStream) must reject null input stream."),
|
"read(InputStream) must reject null input stream."),
|
||||||
() -> assertThrows(NullPointerException.class,
|
() -> assertThrows(NullPointerException.class,
|
||||||
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
"read(InputStream, int) must reject null input stream."));
|
"read(InputStream, int) must reject null input stream."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((Path) null),
|
||||||
|
"readCompiled(Path) must reject null path."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((Path) null, 0),
|
||||||
|
"readCompiled(Path, int) must reject null path."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((String) null),
|
||||||
|
"readCompiled(String) must reject null file name."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((String) null, 0),
|
||||||
|
"readCompiled(String, int) must reject null file name."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null),
|
||||||
|
"readCompiled(InputStream) must reject null input stream."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null, 0),
|
||||||
|
"readCompiled(InputStream, int) must reject null input stream."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(new byte[0]), 0,
|
||||||
|
null),
|
||||||
|
"Injected compiled read must reject a null command compiler."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that direct compiled reading stores compiled commands in final
|
||||||
|
* nodes and shares one version 7 table object across repeated slots.
|
||||||
|
*
|
||||||
|
* @throws IOException if test I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should materialize version 7 directly as shared compiled commands")
|
||||||
|
void shouldMaterializeVersionSevenDirectlyAsSharedCompiledCommands() throws IOException {
|
||||||
|
final FrequencyTrie<String> sourceTrie = sharedPatchTrie();
|
||||||
|
final byte[] artifactBytes = writeCompressed(sourceTrie);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO
|
||||||
|
.readCompiled(new ByteArrayInputStream(artifactBytes));
|
||||||
|
final FrequencyTrie<String> stringTrie = StemmerPatchTrieBinaryIO
|
||||||
|
.read(new ByteArrayInputStream(artifactBytes));
|
||||||
|
final CompiledNode<CompiledPatchCommand> suffixBNode = compiledTrie.root().findChild('b');
|
||||||
|
final CompiledNode<CompiledPatchCommand> abNode = suffixBNode.findChild('a');
|
||||||
|
final CompiledNode<CompiledPatchCommand> cbNode = suffixBNode.findChild('c');
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()),
|
||||||
|
() -> assertEquals(sourceTrie.size(), compiledTrie.size()),
|
||||||
|
() -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()),
|
||||||
|
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
|
||||||
|
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
|
||||||
|
() -> assertEquals("a", compiledTrie.get("ab").apply("ab")),
|
||||||
|
() -> assertEquals("c", compiledTrie.get("cb").apply("cb")),
|
||||||
|
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")),
|
||||||
|
() -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")),
|
||||||
|
() -> assertInstanceOf(String.class, stringTrie.get("ab")),
|
||||||
|
() -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that version 7 direct loading compiles once per distinct value
|
||||||
|
* table entry.
|
||||||
|
*
|
||||||
|
* @throws IOException if test I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should compile each version 7 table entry once")
|
||||||
|
void shouldCompileEachVersionSevenTableEntryOnce() throws IOException {
|
||||||
|
final FrequencyTrie<String> sourceTrie = sharedPatchTrie();
|
||||||
|
final byte[] artifactBytes = writeCompressed(sourceTrie);
|
||||||
|
final AtomicInteger compilationCount = new AtomicInteger();
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO.readCompiled(
|
||||||
|
new ByteArrayInputStream(artifactBytes), -1, (serializedPatch, traversalDirection) -> {
|
||||||
|
compilationCount.incrementAndGet();
|
||||||
|
return CompiledPatchCommand.compile(serializedPatch, traversalDirection);
|
||||||
|
});
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(2, compilationCount.get()),
|
||||||
|
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
|
||||||
|
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the direct historical reader compiles repeated inline
|
||||||
|
* commands once through its reader-local compatibility cache.
|
||||||
|
*
|
||||||
|
* @throws IOException if test I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should compile repeated version 6 inline commands once")
|
||||||
|
void shouldCompileRepeatedVersionSixInlineCommandsOnce() throws IOException {
|
||||||
|
final String serializedPatch = PatchCommandEncoder.builder().build().encode("ab", "a");
|
||||||
|
final byte[] artifactBytes = createVersionSixArtifactWithRepeatedPatch(serializedPatch);
|
||||||
|
final AtomicInteger compilationCount = new AtomicInteger();
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieBinaryIO.readCompiled(
|
||||||
|
new ByteArrayInputStream(artifactBytes), -1, (patch, traversalDirection) -> {
|
||||||
|
compilationCount.incrementAndGet();
|
||||||
|
return CompiledPatchCommand.compile(patch, traversalDirection);
|
||||||
|
});
|
||||||
|
final CompiledPatchCommand rootCommand = compiledTrie.root().orderedValues()[0];
|
||||||
|
final CompiledPatchCommand childCommand = compiledTrie.root().findChild('a').orderedValues()[0];
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(1, compilationCount.get()),
|
||||||
|
() -> assertSame(rootCommand, childCommand),
|
||||||
|
() -> assertEquals(6, compiledTrie.metadata().formatVersion()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that invalid persisted patch commands are rejected only by the
|
||||||
|
* direct compiled path and retain their validation cause.
|
||||||
|
*
|
||||||
|
* @throws IOException if test setup I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should wrap invalid compiled patch commands as IOException")
|
||||||
|
void shouldWrapInvalidCompiledPatchCommandsAsIOException() throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings
|
||||||
|
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("invalid", "Zz");
|
||||||
|
final byte[] artifactBytes = writeCompressed(builder.build());
|
||||||
|
|
||||||
|
final FrequencyTrie<String> stringTrie = StemmerPatchTrieBinaryIO
|
||||||
|
.read(new ByteArrayInputStream(artifactBytes));
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes)));
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals("Zz", stringTrie.get("invalid")),
|
||||||
|
() -> assertTrue(exception.getMessage().contains("Zz")),
|
||||||
|
() -> assertTrue(exception.getMessage().contains("BACKWARD")),
|
||||||
|
() -> assertInstanceOf(IllegalArgumentException.class, exception.getCause()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that an invalid command exactly at the diagnostic boundary remains
|
||||||
|
* complete and does not receive a truncation marker.
|
||||||
|
*
|
||||||
|
* @throws IOException if test setup I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should retain an invalid command exactly at the diagnostic boundary")
|
||||||
|
void shouldRetainInvalidCommandAtDiagnosticBoundary() throws IOException {
|
||||||
|
final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY);
|
||||||
|
|
||||||
|
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, invalidPatch);
|
||||||
|
|
||||||
|
assertAll(() -> assertTrue(exception.getMessage().contains(invalidPatch),
|
||||||
|
"The exact-boundary command must remain complete."),
|
||||||
|
() -> assertFalse(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
|
||||||
|
"The exact-boundary command must not be marked as truncated."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that an invalid command one character beyond the diagnostic
|
||||||
|
* boundary is truncated at the boundary and reports its original length.
|
||||||
|
*
|
||||||
|
* @throws IOException if test setup I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should truncate an invalid command immediately above the diagnostic boundary")
|
||||||
|
void shouldTruncateInvalidCommandAboveDiagnosticBoundary() throws IOException {
|
||||||
|
final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY + 1);
|
||||||
|
final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY)
|
||||||
|
+ DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')';
|
||||||
|
|
||||||
|
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic);
|
||||||
|
|
||||||
|
assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
|
||||||
|
"The boundary-plus-one command must be marked as truncated."),
|
||||||
|
() -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"),
|
||||||
|
"The complete boundary-plus-one command must not appear in the diagnostic."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that substantially oversized invalid commands produce bounded
|
||||||
|
* diagnostics containing only the retained prefix, truncation marker, and
|
||||||
|
* original length.
|
||||||
|
*
|
||||||
|
* @throws IOException if test setup I/O fails unexpectedly
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should bound diagnostics for substantially oversized invalid commands")
|
||||||
|
void shouldBoundDiagnosticForOversizedInvalidCommand() throws IOException {
|
||||||
|
final String invalidPatch = "Z".repeat(512);
|
||||||
|
final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY)
|
||||||
|
+ DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')';
|
||||||
|
|
||||||
|
final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic);
|
||||||
|
|
||||||
|
assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER),
|
||||||
|
"The oversized command must be marked as truncated."),
|
||||||
|
() -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"),
|
||||||
|
"The complete oversized command must not appear in the diagnostic."),
|
||||||
|
() -> assertTrue(exception.getMessage().length() < invalidPatch.length(),
|
||||||
|
"The diagnostic must remain shorter than the oversized persisted command."));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -614,6 +825,127 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds a version 7 trie with equal patch strings on distinct nonmergeable
|
||||||
|
* nodes and one additional distinct patch.
|
||||||
|
*
|
||||||
|
* @return representative patch-command trie
|
||||||
|
*/
|
||||||
|
private static FrequencyTrie<String> sharedPatchTrie() {
|
||||||
|
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||||
|
final String sharedPatch = encoder.encode("ab", "a");
|
||||||
|
final String longerDeletionPatch = encoder.encode("xab", "x");
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("ab", new String(sharedPatch), 2);
|
||||||
|
builder.put("xab", longerDeletionPatch);
|
||||||
|
builder.put("cb", new String(sharedPatch), 3);
|
||||||
|
builder.put("ycb", new String(longerDeletionPatch));
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serializes one String-valued trie through the production compressed writer.
|
||||||
|
*
|
||||||
|
* @param trie source trie
|
||||||
|
* @return compressed artifact bytes
|
||||||
|
* @throws IOException if writing fails
|
||||||
|
*/
|
||||||
|
private static byte[] writeCompressed(final FrequencyTrie<String> trie) throws IOException {
|
||||||
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||||
|
StemmerPatchTrieBinaryIO.write(trie, outputStream);
|
||||||
|
return outputStream.toByteArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads an artifact through an injected rejecting compiler and verifies the
|
||||||
|
* exact bounded trust-boundary diagnostic independently of patch-parser
|
||||||
|
* compatibility behavior for odd command lengths.
|
||||||
|
*
|
||||||
|
* @param invalidPatch invalid serialized patch command
|
||||||
|
* @param expectedDiagnostic complete or bounded command representation expected
|
||||||
|
* inside the diagnostic
|
||||||
|
* @return thrown I/O exception for additional boundary-specific assertions
|
||||||
|
* @throws IOException if fixture serialization fails unexpectedly
|
||||||
|
*/
|
||||||
|
private static IOException assertInvalidCompiledPatchDiagnostic(final String invalidPatch,
|
||||||
|
final String expectedDiagnostic) throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(
|
||||||
|
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("invalid", invalidPatch);
|
||||||
|
final byte[] artifactBytes = writeCompressed(builder.build());
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes), -1,
|
||||||
|
(serializedPatch, traversalDirection) -> {
|
||||||
|
throw new IllegalArgumentException("Injected invalid persisted command.");
|
||||||
|
}),
|
||||||
|
"Direct compiled loading must translate an invalid persisted command to IOException.");
|
||||||
|
final String expectedMessage = "Invalid persisted patch command '" + expectedDiagnostic
|
||||||
|
+ "' for traversal direction BACKWARD.";
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(IOException.class, exception.getClass(),
|
||||||
|
"IOException must be the exact externally visible exception type."),
|
||||||
|
() -> assertEquals(expectedMessage, exception.getMessage(),
|
||||||
|
"The invalid-command diagnostic must match the bounded production contract."),
|
||||||
|
() -> assertNotNull(exception.getCause(), "The validation cause must be retained."),
|
||||||
|
() -> assertEquals(IllegalArgumentException.class, exception.getCause().getClass(),
|
||||||
|
"The original patch-command validation failure must remain the cause."));
|
||||||
|
return exception;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a valid version 6 artifact containing the same inline patch command
|
||||||
|
* in two distinct node slots.
|
||||||
|
*
|
||||||
|
* @param serializedPatch repeated serialized patch command
|
||||||
|
* @return compressed historical artifact bytes
|
||||||
|
* @throws IOException if fixture creation fails
|
||||||
|
*/
|
||||||
|
private static byte[] createVersionSixArtifactWithRepeatedPatch(final String serializedPatch) throws IOException {
|
||||||
|
final TrieMetadata metadata = new TrieMetadata(6, WordTraversalDirection.BACKWARD,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||||
|
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||||
|
final ByteArrayOutputStream rawOutputStream = new ByteArrayOutputStream();
|
||||||
|
try (DataOutputStream dataOutput = new DataOutputStream(rawOutputStream)) {
|
||||||
|
dataOutput.writeInt(0x45475452);
|
||||||
|
dataOutput.writeInt(6);
|
||||||
|
dataOutput.writeInt(2);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeUTF(metadata.toTextBlock());
|
||||||
|
|
||||||
|
dataOutput.writeBoolean(false);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeChar('a');
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeUTF(serializedPatch);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
|
||||||
|
dataOutput.writeBoolean(false);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeUTF(serializedPatch);
|
||||||
|
dataOutput.writeInt(2);
|
||||||
|
}
|
||||||
|
return gzip(rawOutputStream.toByteArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compresses a binary payload using GZip.
|
||||||
|
*
|
||||||
|
* @param payload uncompressed bytes
|
||||||
|
* @return compressed bytes
|
||||||
|
* @throws IOException if compression fails
|
||||||
|
*/
|
||||||
|
private static byte[] gzip(final byte[] payload) throws IOException {
|
||||||
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||||
|
try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream)) {
|
||||||
|
gzipOutputStream.write(payload);
|
||||||
|
}
|
||||||
|
return outputStream.toByteArray();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility method that produces a small GZip-compressed byte array.
|
* Utility method that produces a small GZip-compressed byte array.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -34,10 +34,15 @@ import static org.junit.jupiter.api.Assertions.assertAll;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.Mockito.mockStatic;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
@@ -58,6 +63,7 @@ import java.util.stream.Stream;
|
|||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
|
|
||||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||||
|
import org.egothor.stemmer.trie.CompiledNode;
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.Nested;
|
import org.junit.jupiter.api.Nested;
|
||||||
import org.junit.jupiter.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
@@ -67,6 +73,7 @@ import org.junit.jupiter.api.io.TempDir;
|
|||||||
import org.junit.jupiter.params.ParameterizedTest;
|
import org.junit.jupiter.params.ParameterizedTest;
|
||||||
import org.junit.jupiter.params.provider.Arguments;
|
import org.junit.jupiter.params.provider.Arguments;
|
||||||
import org.junit.jupiter.params.provider.MethodSource;
|
import org.junit.jupiter.params.provider.MethodSource;
|
||||||
|
import org.mockito.MockedStatic;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Professional test suite for {@link StemmerPatchTrieLoader}.
|
* Professional test suite for {@link StemmerPatchTrieLoader}.
|
||||||
@@ -386,6 +393,161 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Structural tests for direct compiled binary-loader delegation.
|
||||||
|
*/
|
||||||
|
@Nested
|
||||||
|
@DisplayName("Compiled binary routing")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("persistence")
|
||||||
|
final class CompiledBinaryRoutingTests {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the path overload returns the direct compiled BinaryIO result
|
||||||
|
* without invoking deprecated String deserialization.
|
||||||
|
*
|
||||||
|
* @throws IOException if the loader unexpectedly reports an I/O failure
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Path compiled loading must use direct compiled BinaryIO")
|
||||||
|
void shouldRouteCompiledPathLoadingDirectly() throws IOException {
|
||||||
|
final Path path = tempDir.resolve("compiled-routing-path.bin.gz");
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
|
||||||
|
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
|
||||||
|
|
||||||
|
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path)).thenReturn(compiledSentinel);
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path)).thenReturn(stringFallback);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader.loadBinaryCompiled(path);
|
||||||
|
|
||||||
|
assertSame(compiledSentinel, actual,
|
||||||
|
"Path loading must return the trie materialized by the direct compiled reader.");
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path), times(1));
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path), never());
|
||||||
|
binaryIo.verifyNoMoreInteractions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the path-and-span overload preserves both arguments while
|
||||||
|
* delegating directly to compiled BinaryIO.
|
||||||
|
*
|
||||||
|
* @throws IOException if the loader unexpectedly reports an I/O failure
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Path and span compiled loading must use direct compiled BinaryIO")
|
||||||
|
void shouldRouteCompiledPathAndSpanLoadingDirectly() throws IOException {
|
||||||
|
final Path path = tempDir.resolve("compiled-routing-path-span.bin.gz");
|
||||||
|
final int maxExpandedIndex = 17;
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
|
||||||
|
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
|
||||||
|
|
||||||
|
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex))
|
||||||
|
.thenReturn(compiledSentinel);
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex)).thenReturn(stringFallback);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader.loadBinaryCompiled(path,
|
||||||
|
maxExpandedIndex);
|
||||||
|
|
||||||
|
assertSame(compiledSentinel, actual,
|
||||||
|
"Path-and-span loading must return the trie materialized by the direct compiled reader.");
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex), times(1));
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex), never());
|
||||||
|
binaryIo.verifyNoMoreInteractions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the file-name overload returns the direct compiled BinaryIO
|
||||||
|
* result without invoking deprecated String deserialization.
|
||||||
|
*
|
||||||
|
* @throws IOException if the loader unexpectedly reports an I/O failure
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("String compiled loading must use direct compiled BinaryIO")
|
||||||
|
void shouldRouteCompiledStringLoadingDirectly() throws IOException {
|
||||||
|
final String fileName = tempDir.resolve("compiled-routing-string.bin.gz").toString();
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
|
||||||
|
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
|
||||||
|
|
||||||
|
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName)).thenReturn(compiledSentinel);
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName)).thenReturn(stringFallback);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(fileName);
|
||||||
|
|
||||||
|
assertSame(compiledSentinel, actual,
|
||||||
|
"String loading must return the trie materialized by the direct compiled reader.");
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName), times(1));
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName), never());
|
||||||
|
binaryIo.verifyNoMoreInteractions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the file-name-and-span overload preserves both arguments
|
||||||
|
* while delegating directly to compiled BinaryIO.
|
||||||
|
*
|
||||||
|
* @throws IOException if the loader unexpectedly reports an I/O failure
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("String and span compiled loading must use direct compiled BinaryIO")
|
||||||
|
void shouldRouteCompiledStringAndSpanLoadingDirectly() throws IOException {
|
||||||
|
final String fileName = tempDir.resolve("compiled-routing-string-span.bin.gz").toString();
|
||||||
|
final int maxExpandedIndex = 17;
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
|
||||||
|
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
|
||||||
|
|
||||||
|
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex))
|
||||||
|
.thenReturn(compiledSentinel);
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex))
|
||||||
|
.thenReturn(stringFallback);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(fileName, maxExpandedIndex);
|
||||||
|
|
||||||
|
assertSame(compiledSentinel, actual,
|
||||||
|
"String-and-span loading must return the trie materialized by the direct compiled reader.");
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex), times(1));
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex), never());
|
||||||
|
binaryIo.verifyNoMoreInteractions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the stream overload preserves stream identity while
|
||||||
|
* delegating directly to compiled BinaryIO.
|
||||||
|
*
|
||||||
|
* @throws IOException if the loader unexpectedly reports an I/O failure
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("InputStream compiled loading must use direct compiled BinaryIO")
|
||||||
|
void shouldRouteCompiledInputStreamLoadingDirectly() throws IOException {
|
||||||
|
final InputStream inputStream = new ByteArrayInputStream(new byte[] { 1, 2, 3 });
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledSentinel = compiledRoutingSentinel();
|
||||||
|
final FrequencyTrie<String> stringFallback = stringRoutingFallback();
|
||||||
|
|
||||||
|
try (MockedStatic<StemmerPatchTrieBinaryIO> binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) {
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream)).thenReturn(compiledSentinel);
|
||||||
|
binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(inputStream)).thenReturn(stringFallback);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> actual = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(inputStream);
|
||||||
|
|
||||||
|
assertSame(compiledSentinel, actual,
|
||||||
|
"Stream loading must return the trie materialized by the direct compiled reader.");
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream), times(1));
|
||||||
|
binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(inputStream), never());
|
||||||
|
binaryIo.verifyNoMoreInteractions();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Focused internal loader behavior tests.
|
* Focused internal loader behavior tests.
|
||||||
*/
|
*/
|
||||||
@@ -707,6 +869,54 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that public compiled binary loading materializes final command
|
||||||
|
* arrays directly and preserves shared table identity, counts, metadata, and
|
||||||
|
* canonical graph size.
|
||||||
|
*
|
||||||
|
* @throws IOException if writing or reading fails
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Compiled binary loading should materialize one final shared command graph")
|
||||||
|
void shouldMaterializeOneFinalSharedCommandGraphFromBinary() throws IOException {
|
||||||
|
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||||
|
final String sharedPatch = encoder.encode("ab", "a");
|
||||||
|
final String longerDeletionPatch = encoder.encode("xab", "x");
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings
|
||||||
|
.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("ab", new String(sharedPatch), 2);
|
||||||
|
builder.put("xab", longerDeletionPatch);
|
||||||
|
builder.put("cb", new String(sharedPatch), 3);
|
||||||
|
builder.put("ycb", new String(longerDeletionPatch));
|
||||||
|
final FrequencyTrie<String> sourceTrie = builder.build();
|
||||||
|
final Path binaryFile = tempDir.resolve("direct-compiled-trie.bin.gz");
|
||||||
|
StemmerPatchTrieLoader.saveBinary(sourceTrie, binaryFile);
|
||||||
|
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledTrie = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(binaryFile);
|
||||||
|
final FrequencyTrie<String> stringTrie = StemmerPatchTrieLoader.loadBinary(binaryFile);
|
||||||
|
final CompiledNode<CompiledPatchCommand> suffixBNode = compiledTrie.root().findChild('b');
|
||||||
|
final CompiledNode<CompiledPatchCommand> abNode = suffixBNode.findChild('a');
|
||||||
|
final CompiledNode<CompiledPatchCommand> cbNode = suffixBNode.findChild('c');
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()),
|
||||||
|
() -> assertEquals(sourceTrie.size(), compiledTrie.size()),
|
||||||
|
() -> assertEquals(sourceTrie.getEntries("ab").get(0).count(),
|
||||||
|
compiledTrie.getEntries("ab").get(0).count()),
|
||||||
|
() -> assertEquals(sourceTrie.getEntries("cb").get(0).count(),
|
||||||
|
compiledTrie.getEntries("cb").get(0).count()),
|
||||||
|
() -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()),
|
||||||
|
() -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")),
|
||||||
|
() -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]),
|
||||||
|
() -> assertEquals("a", compiledTrie.get("ab").apply("ab")),
|
||||||
|
() -> assertEquals("c", compiledTrie.get("cb").apply("cb")),
|
||||||
|
() -> assertEquals("x", compiledTrie.get("xab").apply("xab")),
|
||||||
|
() -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")),
|
||||||
|
() -> assertInstanceOf(String.class, stringTrie.get("ab")),
|
||||||
|
() -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab")));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that binary load overloads with an explicit dense lookup span
|
* Verifies that binary load overloads with an explicit dense lookup span
|
||||||
* preserve trie semantics while honoring the dense-layout override.
|
* preserve trie semantics while honoring the dense-layout override.
|
||||||
@@ -728,9 +938,17 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
|
|
||||||
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
|
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
|
||||||
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
|
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
|
||||||
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX);
|
-1);
|
||||||
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
|
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
|
||||||
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
|
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledDefault = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(binaryFile);
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledDefaultByNegative = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(binaryFile, -1);
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledNoDense = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(binaryFile, 0);
|
||||||
|
final FrequencyTrie<CompiledPatchCommand> compiledStringNoDense = StemmerPatchTrieLoader
|
||||||
|
.loadBinaryCompiled(binaryFile.toString(), 0);
|
||||||
|
|
||||||
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
|
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
|
||||||
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
|
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
|
||||||
@@ -738,11 +956,27 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
|
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
|
||||||
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
|
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
|
||||||
"studying");
|
"studying");
|
||||||
|
assertCompiledTrieSemanticsEqual(original, compiledDefault, "run", "running", "runner", "cities",
|
||||||
|
"studying");
|
||||||
|
assertCompiledTrieSemanticsEqual(original, compiledDefaultByNegative, "run", "running", "runner",
|
||||||
|
"cities", "studying");
|
||||||
|
assertCompiledTrieSemanticsEqual(original, compiledNoDense, "run", "running", "runner", "cities",
|
||||||
|
"studying");
|
||||||
|
assertCompiledTrieSemanticsEqual(original, compiledStringNoDense, "run", "running", "runner", "cities",
|
||||||
|
"studying");
|
||||||
|
|
||||||
|
assertTrue(compiledDefault.root().hasDenseLookup(),
|
||||||
|
"Default compiled loading should use the documented dense lookup span.");
|
||||||
|
assertTrue(compiledDefaultByNegative.root().hasDenseLookup(),
|
||||||
|
"Negative compiled override should use the documented default dense lookup span.");
|
||||||
assertFalse(fromPathNoDense.root().hasDenseLookup(),
|
assertFalse(fromPathNoDense.root().hasDenseLookup(),
|
||||||
"Zero span should disable dense lookup on the loaded root.");
|
"Zero span should disable dense lookup on the loaded root.");
|
||||||
assertFalse(fromStringNoDense.root().hasDenseLookup(),
|
assertFalse(fromStringNoDense.root().hasDenseLookup(),
|
||||||
"Zero span should disable dense lookup on the loaded root.");
|
"Zero span should disable dense lookup on the loaded root.");
|
||||||
|
assertFalse(compiledNoDense.root().hasDenseLookup(),
|
||||||
|
"Zero span should disable dense lookup on the directly compiled root.");
|
||||||
|
assertFalse(compiledStringNoDense.root().hasDenseLookup(),
|
||||||
|
"String-path zero span should disable dense lookup on the directly compiled root.");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -990,6 +1224,36 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
return stems;
|
return stems;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a compiled trie whose identity distinguishes the direct BinaryIO
|
||||||
|
* result from any graph produced through deprecated String loading.
|
||||||
|
*
|
||||||
|
* @return compiled trie sentinel
|
||||||
|
*/
|
||||||
|
private static FrequencyTrie<CompiledPatchCommand> compiledRoutingSentinel() {
|
||||||
|
final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run");
|
||||||
|
final CompiledPatchCommand compiledPatch = CompiledPatchCommand.compile(serializedPatch,
|
||||||
|
WordTraversalDirection.BACKWARD);
|
||||||
|
return new FrequencyTrie.Builder<CompiledPatchCommand>(CompiledPatchCommand[]::new,
|
||||||
|
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE))
|
||||||
|
.put("running", compiledPatch)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a valid String-valued trie that allows the deprecated two-stage route
|
||||||
|
* to execute rather than failing because of incomplete mock setup.
|
||||||
|
*
|
||||||
|
* @return valid String-valued fallback trie
|
||||||
|
*/
|
||||||
|
private static FrequencyTrie<String> stringRoutingFallback() {
|
||||||
|
final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run");
|
||||||
|
return new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE))
|
||||||
|
.put("running", serializedPatch)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies semantic equality of two tries for the supplied words by comparing
|
* Verifies semantic equality of two tries for the supplied words by comparing
|
||||||
* both their raw patch arrays and reconstructed stem sets.
|
* both their raw patch arrays and reconstructed stem sets.
|
||||||
|
|||||||
Reference in New Issue
Block a user