diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrie.java b/src/main/java/org/egothor/stemmer/FrequencyTrie.java index 94b5bde..83a2a1c 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrie.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrie.java @@ -242,6 +242,18 @@ public final class FrequencyTrie { return STREAM_VERSION; } + /** + * Returns whether the supplied metadata identifies a stream format with a + * serialized value table. + * + * @param metadata parsed trie metadata + * @return {@code true} when values are stored in a stream-local table + * @throws NullPointerException if {@code metadata} is {@code null} + */ + /* default */ static boolean usesValueTableFormat(final TrieMetadata metadata) { + return Objects.requireNonNull(metadata, "metadata").formatVersion() >= VALUE_TABLE_VERSION; + } + /** * Receives trie values during visitor-style lookup. * @@ -817,7 +829,37 @@ public final class FrequencyTrie { */ public static FrequencyTrie readFrom(final InputStream inputStream, final IntFunction arrayFactory, final ValueStreamCodec valueCodec, final int maxExpandedIndex) throws IOException { - return CompiledTrieReader.read(inputStream, arrayFactory, valueCodec, maxExpandedIndex); + Objects.requireNonNull(valueCodec, "valueCodec"); + return readFromWithMetadata(inputStream, arrayFactory, + (dataInput, metadata) -> valueCodec.read(dataInput), maxExpandedIndex); + } + + /** + * Reads a compiled trie while allowing value decoding to use already parsed + * trie metadata. + * + *

+ * This package-private path materializes the requested final value type during + * the normal graph read. It does not expose reader state or construct an + * intermediate trie with a different value type. + *

+ * + * @param inputStream source input stream + * @param arrayFactory factory used to create typed value arrays + * @param valueReader metadata-aware value reader + * @param maxExpandedIndex dense lookup span override; zero disables dense + * lookup, negative values use + * {@link #DEFAULT_MAX_EXPANDED_INDEX} + * @param final value type + * @return deserialized compiled trie containing values returned by + * {@code valueReader} + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if reading fails or the binary format is invalid + */ + /* default */ static FrequencyTrie readFromWithMetadata(final InputStream inputStream, + final IntFunction arrayFactory, final MetadataValueStreamReader valueReader, + final int maxExpandedIndex) throws IOException { + return CompiledTrieReader.read(inputStream, arrayFactory, valueReader, maxExpandedIndex); } /** @@ -1044,10 +1086,10 @@ public final class FrequencyTrie { private static final class CompiledTrieReader { private static FrequencyTrie read(final InputStream inputStream, final IntFunction arrayFactory, - final ValueStreamCodec valueCodec, final int maxExpandedIndex) throws IOException { + final MetadataValueStreamReader valueReader, final int maxExpandedIndex) throws IOException { Objects.requireNonNull(inputStream, "inputStream"); Objects.requireNonNull(arrayFactory, "arrayFactory"); - Objects.requireNonNull(valueCodec, "valueCodec"); + Objects.requireNonNull(valueReader, "valueReader"); if (maxExpandedIndex < -1) { throw new IllegalArgumentException("maxExpandedIndex must be >= -1."); } @@ -1074,11 +1116,12 @@ public final class FrequencyTrie { } final TrieMetadata sourceMetadata = readMetadata(dataInput, version); - final V[] valueTable = version >= VALUE_TABLE_VERSION ? readValueTable(dataInput, arrayFactory, valueCodec) + final V[] valueTable = version >= VALUE_TABLE_VERSION + ? readValueTable(dataInput, arrayFactory, valueReader, sourceMetadata) : null; final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX; - final CompiledNode[] nodes = readNodes(dataInput, arrayFactory, valueCodec, valueTable, nodeCount, - effectiveMaxExpandedIndex, version); + final CompiledNode[] nodes = readNodes(dataInput, arrayFactory, valueReader, sourceMetadata, valueTable, + nodeCount, effectiveMaxExpandedIndex, version); final CompiledNode rootNode = nodes[rootNodeId]; if (LOGGER.isLoggable(Level.FINE)) { @@ -1099,13 +1142,14 @@ public final class FrequencyTrie { * * @param dataInput input stream * @param arrayFactory typed-array factory - * @param valueCodec codec responsible for value decoding + * @param valueReader metadata-aware value reader + * @param metadata parsed trie metadata * @param value type * @return decoded values in table-index order * @throws IOException if the count is negative or value decoding fails */ private static V[] readValueTable(final DataInputStream dataInput, final IntFunction arrayFactory, - final ValueStreamCodec valueCodec) throws IOException { + final MetadataValueStreamReader valueReader, final TrieMetadata metadata) throws IOException { final int distinctValueCount = dataInput.readInt(); if (distinctValueCount < 0) { throw new IOException("Negative distinct value count: " + distinctValueCount); @@ -1113,7 +1157,7 @@ public final class FrequencyTrie { final V[] valueTable = arrayFactory.apply(distinctValueCount); for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) { - valueTable[valueIndex] = valueCodec.read(dataInput); + valueTable[valueIndex] = valueReader.read(dataInput, metadata); } return valueTable; } @@ -1181,8 +1225,9 @@ public final class FrequencyTrie { } private static CompiledNode[] readNodes(final DataInputStream dataInput, - final IntFunction arrayFactory, final ValueStreamCodec valueCodec, final V[] valueTable, - final int nodeCount, final int maxExpandedIndex, final int version) throws IOException { + final IntFunction arrayFactory, final MetadataValueStreamReader valueReader, + final TrieMetadata metadata, final V[] valueTable, final int nodeCount, final int maxExpandedIndex, + final int version) throws IOException { final char[][] edgeLabelsByNode = new char[nodeCount][]; final int[][] childNodeIdsByNode = new int[nodeCount][]; @SuppressWarnings("unchecked") @@ -1234,7 +1279,7 @@ public final class FrequencyTrie { } orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex]; } else { - orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput); + orderedValuesByNode[nodeIndex][valueIndex] = valueReader.read(dataInput, metadata); } orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt(); if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) { @@ -1980,6 +2025,32 @@ public final class FrequencyTrie { } } + /** + * Reads one final trie value using metadata parsed from the same binary stream. + * + *

+ * Implementations are invoked only during deserialization and are not retained + * by the resulting trie. Returned values are stored directly in final compiled + * node arrays. + *

+ * + * @param final value type + */ + /* default */ + @FunctionalInterface + interface MetadataValueStreamReader { + + /** + * Reads and materializes one final value. + * + * @param dataInput source data input + * @param metadata already parsed trie metadata + * @return final value to store directly in compiled nodes + * @throws IOException if reading or materialization fails + */ + V read(DataInputStream dataInput, TrieMetadata metadata) throws IOException; + } + /** * Codec used to persist values stored in the trie. * diff --git a/src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java b/src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java index 4f16d1c..0401594 100644 --- a/src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java +++ b/src/main/java/org/egothor/stemmer/StemmerPatchTrieBinaryIO.java @@ -39,7 +39,10 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; @@ -53,6 +56,8 @@ import java.util.zip.GZIPOutputStream; * patch commands represented as {@link String}. The serialized trie payload is * the native binary format of {@link FrequencyTrie}, wrapped in GZip * compression. + * Binary reads can either preserve those serialized strings or materialize + * {@link CompiledPatchCommand} values directly in the final trie nodes. * *

* The helper centralizes the codec and compression details so that higher-level @@ -71,6 +76,21 @@ public final class StemmerPatchTrieBinaryIO { */ private static final FrequencyTrie.ValueStreamCodec STRING_CODEC = new StringValueStreamCodec(); + /** + * Maximum serialized patch-command length included in validation diagnostics. + */ + private static final int MAX_DIAGNOSTIC_PATCH_LENGTH = 128; + + /** + * Null-check parameter name for filesystem paths. + */ + private static final String PATH_PARAMETER = "path"; + + /** + * Null-check parameter name for filesystem path strings. + */ + private static final String FILE_NAME_PARAMETER = "fileName"; + /** * Utility class. */ @@ -87,7 +107,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static FrequencyTrie read(final Path path) throws IOException { - Objects.requireNonNull(path, "path"); + Objects.requireNonNull(path, PATH_PARAMETER); try (InputStream fileInputStream = Files.newInputStream(path)) { return read(fileInputStream); @@ -110,7 +130,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static FrequencyTrie read(final Path path, final int maxExpandedIndex) throws IOException { - Objects.requireNonNull(path, "path"); + Objects.requireNonNull(path, PATH_PARAMETER); try (InputStream fileInputStream = Files.newInputStream(path)) { return read(fileInputStream, maxExpandedIndex); @@ -127,7 +147,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static FrequencyTrie read(final String fileName) throws IOException { - Objects.requireNonNull(fileName, "fileName"); + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); return read(Path.of(fileName)); } @@ -147,7 +167,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static FrequencyTrie read(final String fileName, final int maxExpandedIndex) throws IOException { - Objects.requireNonNull(fileName, "fileName"); + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); return read(Path.of(fileName), maxExpandedIndex); } @@ -204,6 +224,149 @@ public final class StemmerPatchTrieBinaryIO { } } + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * a filesystem path. + * + * @param path source file + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code path} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final Path path) throws IOException { + Objects.requireNonNull(path, PATH_PARAMETER); + + try (InputStream fileInputStream = Files.newInputStream(path)) { + return readCompiled(fileInputStream); + } + } + + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * a filesystem path with a dense child lookup span override. + * + * @param path source file + * @param maxExpandedIndex dense lookup span override; negative values use + * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code path} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final Path path, + final int maxExpandedIndex) + throws IOException { + Objects.requireNonNull(path, PATH_PARAMETER); + + try (InputStream fileInputStream = Files.newInputStream(path)) { + return readCompiled(fileInputStream, maxExpandedIndex); + } + } + + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * a filesystem path string. + * + * @param fileName source file name or path string + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code fileName} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final String fileName) throws IOException { + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); + return readCompiled(Path.of(fileName)); + } + + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * a filesystem path string with a dense child lookup span override. + * + * @param fileName source file name or path string + * @param maxExpandedIndex dense lookup span override; negative values use + * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code fileName} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final String fileName, + final int maxExpandedIndex) + throws IOException { + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); + return readCompiled(Path.of(fileName), maxExpandedIndex); + } + + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * an input stream. + * + * @param inputStream source stream + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code inputStream} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final InputStream inputStream) + throws IOException { + return readCompiled(inputStream, -1); + } + + /** + * Reads a compressed binary patch-command trie directly as compiled values from + * an input stream with a dense child lookup span override. + * + * @param inputStream source stream + * @param maxExpandedIndex dense lookup span override; negative values use + * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if {@code inputStream} is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final InputStream inputStream, + final int maxExpandedIndex) throws IOException { + return readCompiled(inputStream, maxExpandedIndex, CompiledPatchCommand::compile); + } + + /** + * Reads a compressed binary patch-command trie using a caller-supplied command + * compiler. + * + *

+ * This package-private seam permits deterministic compilation-count testing + * without global counters. Production callers use + * {@link CompiledPatchCommand#compile(String, WordTraversalDirection)}. + *

+ * + * @param inputStream source stream + * @param maxExpandedIndex dense lookup span override + * @param commandCompiler compiler for one serialized command and traversal + * direction + * @return directly materialized compiled patch-command trie + * @throws NullPointerException if any argument is {@code null} + * @throws IOException if reading, decompression, or command compilation + * fails + */ + /* default */ static FrequencyTrie readCompiled(final InputStream inputStream, + final int maxExpandedIndex, + final BiFunction commandCompiler) + throws IOException { + Objects.requireNonNull(inputStream, "inputStream"); + Objects.requireNonNull(commandCompiler, "commandCompiler"); + final CompiledPatchValueReader valueReader = new CompiledPatchValueReader(commandCompiler); + + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream)); + DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) { + final FrequencyTrie trie = FrequencyTrie.readFromWithMetadata(dataInputStream, + CompiledPatchCommand[]::new, valueReader, maxExpandedIndex); + + LOGGER.log(Level.FINE, "Read compressed binary stemmer trie directly as compiled patch commands."); + return trie; + } + } + /** * Reads only metadata from a GZip-compressed binary patch-command trie stored * at a filesystem path. @@ -214,7 +377,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static TrieMetadata readMetadata(final Path path) throws IOException { - Objects.requireNonNull(path, "path"); + Objects.requireNonNull(path, PATH_PARAMETER); return read(path).metadata(); } @@ -228,7 +391,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if reading or decompression fails */ public static TrieMetadata readMetadata(final String fileName) throws IOException { - Objects.requireNonNull(fileName, "fileName"); + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); return readMetadata(Path.of(fileName)); } @@ -256,7 +419,7 @@ public final class StemmerPatchTrieBinaryIO { */ public static void write(final FrequencyTrie trie, final Path path) throws IOException { Objects.requireNonNull(trie, "trie"); - Objects.requireNonNull(path, "path"); + Objects.requireNonNull(path, PATH_PARAMETER); final Path parent = path.toAbsolutePath().getParent(); if (parent != null) { @@ -278,7 +441,7 @@ public final class StemmerPatchTrieBinaryIO { * @throws IOException if writing fails */ public static void write(final FrequencyTrie trie, final String fileName) throws IOException { - Objects.requireNonNull(fileName, "fileName"); + Objects.requireNonNull(fileName, FILE_NAME_PARAMETER); write(trie, Path.of(fileName)); } @@ -323,4 +486,97 @@ public final class StemmerPatchTrieBinaryIO { return dataInput.readUTF(); } } + + /** + * Metadata-aware reader that compiles serialized patch commands into final trie + * values. + * + *

+ * Version 7 value tables already contain distinct serialized values, so each + * entry is compiled directly. Historical inline formats use the reader-local + * equality cache to compile repeated serialized commands once. Neither the cache + * nor this reader is retained after trie loading. + *

+ */ + private static final class CompiledPatchValueReader + implements FrequencyTrie.MetadataValueStreamReader { + + /** + * Compiler used to materialize one final command. + */ + private final BiFunction commandCompiler; + + /** + * Compatibility cache used only by historical inline-value streams. + */ + private final Map legacyCompiledCommands = new HashMap<>(); + + /** + * Creates one reader with a caller-supplied compiler. + * + * @param commandCompiler compiler for serialized patch commands + */ + private CompiledPatchValueReader( + final BiFunction commandCompiler) { + this.commandCompiler = commandCompiler; + } + + /** + * Reads and compiles one serialized patch command. + * + * @param dataInput source data input + * @param metadata parsed trie metadata + * @return final compiled patch command + * @throws IOException if reading or command compilation fails + */ + @Override + public CompiledPatchCommand read(final DataInputStream dataInput, final TrieMetadata metadata) + throws IOException { + final String serializedPatch = dataInput.readUTF(); + if (FrequencyTrie.usesValueTableFormat(metadata)) { + return compile(serializedPatch, metadata.traversalDirection()); + } + + final CompiledPatchCommand cachedCommand = this.legacyCompiledCommands.get(serializedPatch); + if (cachedCommand != null) { + return cachedCommand; + } + final CompiledPatchCommand compiledCommand = compile(serializedPatch, metadata.traversalDirection()); + this.legacyCompiledCommands.put(serializedPatch, compiledCommand); + return compiledCommand; + } + + /** + * Compiles one serialized command and converts validation failures to + * trust-boundary {@link IOException} instances. + * + * @param serializedPatch serialized patch command + * @param traversalDirection traversal direction from persisted metadata + * @return compiled patch command + * @throws IOException if the serialized command is invalid + */ + private CompiledPatchCommand compile(final String serializedPatch, + final WordTraversalDirection traversalDirection) throws IOException { + try { + return this.commandCompiler.apply(serializedPatch, traversalDirection); + } catch (IllegalArgumentException exception) { + throw new IOException("Invalid persisted patch command '" + boundedPatch(serializedPatch) + + "' for traversal direction " + traversalDirection + '.', exception); + } + } + } + + /** + * Returns a safely bounded patch-command representation for diagnostics. + * + * @param serializedPatch serialized patch command + * @return complete or length-bounded diagnostic representation + */ + private static String boundedPatch(final String serializedPatch) { + if (serializedPatch.length() <= MAX_DIAGNOSTIC_PATCH_LENGTH) { + return serializedPatch; + } + return serializedPatch.substring(0, MAX_DIAGNOSTIC_PATCH_LENGTH) + "... (length " + + serializedPatch.length() + ')'; + } } diff --git a/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java b/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java index 0697e42..d3fc6d4 100644 --- a/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java +++ b/src/main/java/org/egothor/stemmer/StemmerPatchTrieLoader.java @@ -1311,6 +1311,12 @@ public final class StemmerPatchTrieLoader { * Loads a GZip-compressed binary patch-command trie from a filesystem path and * returns runtime-specialized compiled patch values. * + *

+ * Serialized patch commands are compiled while the binary graph is read, so the + * returned trie is materialized directly without an intermediate + * {@code FrequencyTrie} or a second node graph. + *

+ * * @param path path to the compressed binary trie file * @return compiled patch-command trie with runtime-specialized values * @throws NullPointerException if {@code path} is {@code null} @@ -1318,7 +1324,8 @@ public final class StemmerPatchTrieLoader { * read */ public static FrequencyTrie loadBinaryCompiled(final Path path) throws IOException { - return compilePatchTrie(loadBinary(path)); + Objects.requireNonNull(path, PARAMETER_PATH); + return StemmerPatchTrieBinaryIO.readCompiled(path); } /** @@ -1351,6 +1358,11 @@ public final class StemmerPatchTrieLoader { * a custom dense lookup span override and returns runtime-specialized compiled * patch values. * + *

+ * Serialized patch commands are compiled directly into the final node graph. + * The dense lookup override is applied during that graph materialization. + *

+ * * @param path path to the compressed binary trie file * @param maxExpandedIndex dense lookup span override; negative values use * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} @@ -1361,7 +1373,8 @@ public final class StemmerPatchTrieLoader { */ public static FrequencyTrie loadBinaryCompiled(final Path path, final int maxExpandedIndex) throws IOException { - return compilePatchTrie(loadBinary(path, maxExpandedIndex)); + Objects.requireNonNull(path, PARAMETER_PATH); + return StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex); } /** @@ -1387,6 +1400,11 @@ public final class StemmerPatchTrieLoader { * Loads a GZip-compressed binary patch-command trie from a filesystem path * string and returns runtime-specialized compiled patch values. * + *

+ * Serialized patch commands are compiled while the binary graph is read, so no + * intermediate String-valued trie is constructed. + *

+ * * @param fileName file name or path string * @return compiled patch-command trie with runtime-specialized values * @throws NullPointerException if {@code fileName} is {@code null} @@ -1394,7 +1412,8 @@ public final class StemmerPatchTrieLoader { * read */ public static FrequencyTrie loadBinaryCompiled(final String fileName) throws IOException { - return compilePatchTrie(loadBinary(fileName)); + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return StemmerPatchTrieBinaryIO.readCompiled(fileName); } /** @@ -1428,6 +1447,11 @@ public final class StemmerPatchTrieLoader { * using a custom dense lookup span override and returns runtime-specialized * compiled patch values. * + *

+ * Serialized patch commands are compiled directly into the final node graph. + * The dense lookup override is applied during that graph materialization. + *

+ * * @param fileName file name or path string * @param maxExpandedIndex dense lookup span override; negative values use * {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX} @@ -1438,7 +1462,8 @@ public final class StemmerPatchTrieLoader { */ public static FrequencyTrie loadBinaryCompiled(final String fileName, final int maxExpandedIndex) throws IOException { - return compilePatchTrie(loadBinary(fileName, maxExpandedIndex)); + Objects.requireNonNull(fileName, FILENAME_REQUIRED); + return StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex); } /** @@ -1462,6 +1487,12 @@ public final class StemmerPatchTrieLoader { * Loads a GZip-compressed binary patch-command trie from an input stream and * returns runtime-specialized compiled patch values. * + *

+ * Serialized patch commands are compiled while the binary graph is read, so the + * returned trie is materialized directly without an intermediate + * {@code FrequencyTrie} or graph-mapping pass. + *

+ * * @param inputStream source input stream * @return compiled patch-command trie with runtime-specialized values * @throws NullPointerException if {@code inputStream} is {@code null} @@ -1469,7 +1500,8 @@ public final class StemmerPatchTrieLoader { */ public static FrequencyTrie loadBinaryCompiled(final InputStream inputStream) throws IOException { - return compilePatchTrie(loadBinary(inputStream)); + Objects.requireNonNull(inputStream, "inputStream"); + return StemmerPatchTrieBinaryIO.readCompiled(inputStream); } /** diff --git a/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java b/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java index 24e2fb5..73d0ea1 100644 --- a/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java +++ b/src/test/java/org/egothor/stemmer/CompiledTrieArtifactRegressionTest.java @@ -150,6 +150,25 @@ final class CompiledTrieArtifactRegressionTest { () -> assertGoldenArtifactSemanticProbes(trie, artifactCase)); } + /** + * Verifies that every committed version 6 artifact can be materialized directly + * as compiled commands while preserving representative stemming behavior. + * + * @param artifactCase regression case + * @throws IOException if artifact loading fails + */ + @ParameterizedTest(name = "{0}") + @MethodSource("artifactCases") + @DisplayName("Committed golden artifacts must load directly as compiled commands") + void shouldLoadGoldenArtifactsDirectlyAsCompiledCommands(final ArtifactCase artifactCase) throws IOException { + final byte[] goldenArtifactBytes = RegressionArtifactSupport + .readResourceBytes(artifactCase.goldenArtifactResource()); + final FrequencyTrie trie = StemmerPatchTrieLoader + .loadBinaryCompiled(new ByteArrayInputStream(goldenArtifactBytes)); + + assertGoldenCompiledArtifactSemanticProbes(trie, artifactCase); + } + /** * Verifies in-process determinism independently of the checked-in golden file * by compiling the same dictionary twice and requiring identical artifact @@ -264,6 +283,35 @@ final class CompiledTrieArtifactRegressionTest { } } + /** + * Verifies representative semantic probes against a directly materialized + * compiled-command trie. + * + * @param trie compiled-command trie to inspect + * @param artifactCase regression case providing the expected probes + */ + private static void assertGoldenCompiledArtifactSemanticProbes( + final FrequencyTrie trie, final ArtifactCase artifactCase) { + for (ProbeExpectation probe : artifactCase.probes()) { + final CompiledPatchCommand[] allPatchCommands = trie.getAll(probe.word()); + final CompiledPatchCommand preferredPatchCommand = trie.get(probe.word()); + final String preferredStem = preferredPatchCommand == null ? null : preferredPatchCommand.apply(probe.word()); + final Set allStems = new LinkedHashSet(); + for (CompiledPatchCommand patchCommand : allPatchCommands) { + allStems.add(patchCommand.apply(probe.word())); + } + + assertAll( + () -> assertFalse(allPatchCommands.length == 0, + "Representative probe must produce at least one result for word: " + probe.word()), + () -> assertEquals(probe.preferredStem(), preferredStem, + "Preferred stem mismatch for representative probe word: " + probe.word()), + () -> assertTrue(allStems.containsAll(probe.acceptableStems()), + "All acceptable stems must be present in getAll() for representative probe word: " + + probe.word())); + } + } + /** * Immutable regression case definition. * diff --git a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java index 273105d..c80f5dd 100644 --- a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java +++ b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java @@ -1112,6 +1112,82 @@ class FrequencyTrieTest { () -> assertNull(restored.get("missing"))); } + /** + * Verifies that metadata-aware version 7 reads receive parsed metadata before + * decoding and materialize shared final values directly in compiled nodes. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Metadata-aware version 7 reader materializes shared final values") + void metadataAwareVersionSevenReaderMaterializesSharedFinalValues() throws IOException { + final FrequencyTrie original = sharedValueTrie(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + original.writeTo(outputStream, STRING_CODEC); + final AtomicInteger readCount = new AtomicInteger(); + final AtomicInteger observedMetadataVersion = new AtomicInteger(); + + final FrequencyTrie restored = FrequencyTrie.readFromWithMetadata( + new ByteArrayInputStream(outputStream.toByteArray()), StringBuilder[]::new, + (dataInput, metadata) -> { + observedMetadataVersion.set(metadata.formatVersion()); + readCount.incrementAndGet(); + return new StringBuilder(dataInput.readUTF()); + }, -1); + final CompiledNode suffixBNode = restored.root().findChild('b'); + final CompiledNode abNode = suffixBNode.findChild('a'); + final CompiledNode cbNode = suffixBNode.findChild('c'); + + assertAll(() -> assertEquals(7, observedMetadataVersion.get()), + () -> assertEquals(3, readCount.get()), + () -> assertSame(restored.get("ab"), restored.get("cb")), + () -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]), + () -> assertEquals(StringBuilder[].class, abNode.orderedValues().getClass()), + () -> assertEquals("left", restored.get("xab").toString()), + () -> assertEquals("right", restored.get("ycb").toString())); + } + + /** + * Verifies that metadata-aware reading preserves the inline value layout used + * by every historical stream version from 1 through 6. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Metadata-aware reader supports inline values in versions 1 through 6") + void metadataAwareReaderSupportsInlineValuesInVersionsOneThroughSix() throws IOException { + for (int version = 1; version <= 6; version++) { + final int historicalVersion = version; + final byte[] bytes = createSerializedStream(0x45475452, historicalVersion, 1, 0, + dataOutput -> writeMetadataForHistoricalVersion(dataOutput, historicalVersion), + new NodeWriter[] { dataOutput -> { + if (historicalVersion >= 6) { + dataOutput.writeBoolean(false); + } + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeUTF("inline-" + historicalVersion); + dataOutput.writeInt(1); + } }); + final AtomicInteger readCount = new AtomicInteger(); + final AtomicInteger observedMetadataVersion = new AtomicInteger(); + + final FrequencyTrie trie = FrequencyTrie.readFromWithMetadata( + new ByteArrayInputStream(bytes), StringBuilder[]::new, + (dataInput, metadata) -> { + observedMetadataVersion.set(metadata.formatVersion()); + readCount.incrementAndGet(); + return new StringBuilder(dataInput.readUTF()); + }, -1); + + assertAll(() -> assertEquals(historicalVersion, observedMetadataVersion.get()), + () -> assertEquals(1, readCount.get()), + () -> assertEquals("inline-" + historicalVersion, trie.get("").toString())); + } + } + /** * Verifies fingerprint stability and sensitivity to metadata and trie content. */ @@ -2325,6 +2401,42 @@ class FrequencyTrieTest { dataOutput.writeInt(occurrenceCount); } + /** + * Writes the metadata layout used by one historical stream version. + * + * @param dataOutput output stream + * @param version historical stream version from 1 through 6 + * @throws IOException if writing fails + */ + private static void writeMetadataForHistoricalVersion(final DataOutputStream dataOutput, final int version) + throws IOException { + if (version == 1) { + return; + } + if (version == 2) { + dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal()); + return; + } + + final ReductionSettings reductionSettings = ReductionSettings + .withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); + if (version <= 4) { + dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal()); + dataOutput.writeInt(reductionSettings.reductionMode().ordinal()); + dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent()); + dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio()); + dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal()); + if (version == 4) { + dataOutput.writeInt(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT.ordinal()); + } + return; + } + + final TrieMetadata metadata = new TrieMetadata(version, WordTraversalDirection.BACKWARD, reductionSettings, + DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT); + dataOutput.writeUTF(metadata.toTextBlock()); + } + /** * Writes one synthetic metadata block. */ diff --git a/src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java b/src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java index 37b8e80..9b49dd2 100644 --- a/src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java +++ b/src/test/java/org/egothor/stemmer/StemmerPatchTrieBinaryIOTest.java @@ -32,6 +32,7 @@ package org.egothor.stemmer; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -50,14 +51,18 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.egothor.stemmer.trie.CompiledNode; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Tag; @@ -98,6 +103,17 @@ import org.mockito.MockedStatic; @DisplayName("StemmerPatchTrieBinaryIO") class StemmerPatchTrieBinaryIOTest { + /** + * Maximum invalid patch-command length retained verbatim in production + * diagnostics. + */ + private static final int DIAGNOSTIC_PATCH_BOUNDARY = 128; + + /** + * Marker that introduces the original length after diagnostic truncation. + */ + private static final String DIAGNOSTIC_TRUNCATION_MARKER = "... (length "; + /** * Temporary directory provided by JUnit. */ @@ -323,7 +339,202 @@ class StemmerPatchTrieBinaryIOTest { "read(InputStream) must reject null input stream."), () -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX), - "read(InputStream, int) must reject null input stream.")); + "read(InputStream, int) must reject null input stream."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((Path) null), + "readCompiled(Path) must reject null path."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((Path) null, 0), + "readCompiled(Path, int) must reject null path."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((String) null), + "readCompiled(String) must reject null file name."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((String) null, 0), + "readCompiled(String, int) must reject null file name."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null), + "readCompiled(InputStream) must reject null input stream."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled((InputStream) null, 0), + "readCompiled(InputStream, int) must reject null input stream."), + () -> assertThrows(NullPointerException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(new byte[0]), 0, + null), + "Injected compiled read must reject a null command compiler.")); + } + + /** + * Verifies that direct compiled reading stores compiled commands in final + * nodes and shares one version 7 table object across repeated slots. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @DisplayName("Should materialize version 7 directly as shared compiled commands") + void shouldMaterializeVersionSevenDirectlyAsSharedCompiledCommands() throws IOException { + final FrequencyTrie sourceTrie = sharedPatchTrie(); + final byte[] artifactBytes = writeCompressed(sourceTrie); + + final FrequencyTrie compiledTrie = StemmerPatchTrieBinaryIO + .readCompiled(new ByteArrayInputStream(artifactBytes)); + final FrequencyTrie stringTrie = StemmerPatchTrieBinaryIO + .read(new ByteArrayInputStream(artifactBytes)); + final CompiledNode suffixBNode = compiledTrie.root().findChild('b'); + final CompiledNode abNode = suffixBNode.findChild('a'); + final CompiledNode cbNode = suffixBNode.findChild('c'); + + assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()), + () -> assertEquals(sourceTrie.size(), compiledTrie.size()), + () -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()), + () -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")), + () -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]), + () -> assertEquals("a", compiledTrie.get("ab").apply("ab")), + () -> assertEquals("c", compiledTrie.get("cb").apply("cb")), + () -> assertEquals("x", compiledTrie.get("xab").apply("xab")), + () -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")), + () -> assertInstanceOf(String.class, stringTrie.get("ab")), + () -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab"))); + } + + /** + * Verifies that version 7 direct loading compiles once per distinct value + * table entry. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @DisplayName("Should compile each version 7 table entry once") + void shouldCompileEachVersionSevenTableEntryOnce() throws IOException { + final FrequencyTrie sourceTrie = sharedPatchTrie(); + final byte[] artifactBytes = writeCompressed(sourceTrie); + final AtomicInteger compilationCount = new AtomicInteger(); + + final FrequencyTrie compiledTrie = StemmerPatchTrieBinaryIO.readCompiled( + new ByteArrayInputStream(artifactBytes), -1, (serializedPatch, traversalDirection) -> { + compilationCount.incrementAndGet(); + return CompiledPatchCommand.compile(serializedPatch, traversalDirection); + }); + + assertAll(() -> assertEquals(2, compilationCount.get()), + () -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")), + () -> assertEquals("x", compiledTrie.get("xab").apply("xab"))); + } + + /** + * Verifies that the direct historical reader compiles repeated inline + * commands once through its reader-local compatibility cache. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @DisplayName("Should compile repeated version 6 inline commands once") + void shouldCompileRepeatedVersionSixInlineCommandsOnce() throws IOException { + final String serializedPatch = PatchCommandEncoder.builder().build().encode("ab", "a"); + final byte[] artifactBytes = createVersionSixArtifactWithRepeatedPatch(serializedPatch); + final AtomicInteger compilationCount = new AtomicInteger(); + + final FrequencyTrie compiledTrie = StemmerPatchTrieBinaryIO.readCompiled( + new ByteArrayInputStream(artifactBytes), -1, (patch, traversalDirection) -> { + compilationCount.incrementAndGet(); + return CompiledPatchCommand.compile(patch, traversalDirection); + }); + final CompiledPatchCommand rootCommand = compiledTrie.root().orderedValues()[0]; + final CompiledPatchCommand childCommand = compiledTrie.root().findChild('a').orderedValues()[0]; + + assertAll(() -> assertEquals(1, compilationCount.get()), + () -> assertSame(rootCommand, childCommand), + () -> assertEquals(6, compiledTrie.metadata().formatVersion())); + } + + /** + * Verifies that invalid persisted patch commands are rejected only by the + * direct compiled path and retain their validation cause. + * + * @throws IOException if test setup I/O fails unexpectedly + */ + @Test + @DisplayName("Should wrap invalid compiled patch commands as IOException") + void shouldWrapInvalidCompiledPatchCommandsAsIOException() throws IOException { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder(String[]::new, + ReductionSettings + .withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + builder.put("invalid", "Zz"); + final byte[] artifactBytes = writeCompressed(builder.build()); + + final FrequencyTrie stringTrie = StemmerPatchTrieBinaryIO + .read(new ByteArrayInputStream(artifactBytes)); + final IOException exception = assertThrows(IOException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes))); + + assertAll(() -> assertEquals("Zz", stringTrie.get("invalid")), + () -> assertTrue(exception.getMessage().contains("Zz")), + () -> assertTrue(exception.getMessage().contains("BACKWARD")), + () -> assertInstanceOf(IllegalArgumentException.class, exception.getCause())); + } + + /** + * Verifies that an invalid command exactly at the diagnostic boundary remains + * complete and does not receive a truncation marker. + * + * @throws IOException if test setup I/O fails unexpectedly + */ + @Test + @DisplayName("Should retain an invalid command exactly at the diagnostic boundary") + void shouldRetainInvalidCommandAtDiagnosticBoundary() throws IOException { + final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY); + + final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, invalidPatch); + + assertAll(() -> assertTrue(exception.getMessage().contains(invalidPatch), + "The exact-boundary command must remain complete."), + () -> assertFalse(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER), + "The exact-boundary command must not be marked as truncated.")); + } + + /** + * Verifies that an invalid command one character beyond the diagnostic + * boundary is truncated at the boundary and reports its original length. + * + * @throws IOException if test setup I/O fails unexpectedly + */ + @Test + @DisplayName("Should truncate an invalid command immediately above the diagnostic boundary") + void shouldTruncateInvalidCommandAboveDiagnosticBoundary() throws IOException { + final String invalidPatch = "Z".repeat(DIAGNOSTIC_PATCH_BOUNDARY + 1); + final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY) + + DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')'; + + final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic); + + assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER), + "The boundary-plus-one command must be marked as truncated."), + () -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"), + "The complete boundary-plus-one command must not appear in the diagnostic.")); + } + + /** + * Verifies that substantially oversized invalid commands produce bounded + * diagnostics containing only the retained prefix, truncation marker, and + * original length. + * + * @throws IOException if test setup I/O fails unexpectedly + */ + @Test + @DisplayName("Should bound diagnostics for substantially oversized invalid commands") + void shouldBoundDiagnosticForOversizedInvalidCommand() throws IOException { + final String invalidPatch = "Z".repeat(512); + final String expectedDiagnostic = invalidPatch.substring(0, DIAGNOSTIC_PATCH_BOUNDARY) + + DIAGNOSTIC_TRUNCATION_MARKER + invalidPatch.length() + ')'; + + final IOException exception = assertInvalidCompiledPatchDiagnostic(invalidPatch, expectedDiagnostic); + + assertAll(() -> assertTrue(exception.getMessage().contains(DIAGNOSTIC_TRUNCATION_MARKER), + "The oversized command must be marked as truncated."), + () -> assertFalse(exception.getMessage().contains("'" + invalidPatch + "'"), + "The complete oversized command must not appear in the diagnostic."), + () -> assertTrue(exception.getMessage().length() < invalidPatch.length(), + "The diagnostic must remain shorter than the oversized persisted command.")); } /** @@ -614,6 +825,127 @@ class StemmerPatchTrieBinaryIOTest { } } + /** + * Builds a version 7 trie with equal patch strings on distinct nonmergeable + * nodes and one additional distinct patch. + * + * @return representative patch-command trie + */ + private static FrequencyTrie sharedPatchTrie() { + final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build(); + final String sharedPatch = encoder.encode("ab", "a"); + final String longerDeletionPatch = encoder.encode("xab", "x"); + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder(String[]::new, + ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + builder.put("ab", new String(sharedPatch), 2); + builder.put("xab", longerDeletionPatch); + builder.put("cb", new String(sharedPatch), 3); + builder.put("ycb", new String(longerDeletionPatch)); + return builder.build(); + } + + /** + * Serializes one String-valued trie through the production compressed writer. + * + * @param trie source trie + * @return compressed artifact bytes + * @throws IOException if writing fails + */ + private static byte[] writeCompressed(final FrequencyTrie trie) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + StemmerPatchTrieBinaryIO.write(trie, outputStream); + return outputStream.toByteArray(); + } + + /** + * Loads an artifact through an injected rejecting compiler and verifies the + * exact bounded trust-boundary diagnostic independently of patch-parser + * compatibility behavior for odd command lengths. + * + * @param invalidPatch invalid serialized patch command + * @param expectedDiagnostic complete or bounded command representation expected + * inside the diagnostic + * @return thrown I/O exception for additional boundary-specific assertions + * @throws IOException if fixture serialization fails unexpectedly + */ + private static IOException assertInvalidCompiledPatchDiagnostic(final String invalidPatch, + final String expectedDiagnostic) throws IOException { + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder(String[]::new, + ReductionSettings.withDefaults( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + builder.put("invalid", invalidPatch); + final byte[] artifactBytes = writeCompressed(builder.build()); + final IOException exception = assertThrows(IOException.class, + () -> StemmerPatchTrieBinaryIO.readCompiled(new ByteArrayInputStream(artifactBytes), -1, + (serializedPatch, traversalDirection) -> { + throw new IllegalArgumentException("Injected invalid persisted command."); + }), + "Direct compiled loading must translate an invalid persisted command to IOException."); + final String expectedMessage = "Invalid persisted patch command '" + expectedDiagnostic + + "' for traversal direction BACKWARD."; + + assertAll(() -> assertEquals(IOException.class, exception.getClass(), + "IOException must be the exact externally visible exception type."), + () -> assertEquals(expectedMessage, exception.getMessage(), + "The invalid-command diagnostic must match the bounded production contract."), + () -> assertNotNull(exception.getCause(), "The validation cause must be retained."), + () -> assertEquals(IllegalArgumentException.class, exception.getCause().getClass(), + "The original patch-command validation failure must remain the cause.")); + return exception; + } + + /** + * Creates a valid version 6 artifact containing the same inline patch command + * in two distinct node slots. + * + * @param serializedPatch repeated serialized patch command + * @return compressed historical artifact bytes + * @throws IOException if fixture creation fails + */ + private static byte[] createVersionSixArtifactWithRepeatedPatch(final String serializedPatch) throws IOException { + final TrieMetadata metadata = new TrieMetadata(6, WordTraversalDirection.BACKWARD, + ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS), + DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT); + final ByteArrayOutputStream rawOutputStream = new ByteArrayOutputStream(); + try (DataOutputStream dataOutput = new DataOutputStream(rawOutputStream)) { + dataOutput.writeInt(0x45475452); + dataOutput.writeInt(6); + dataOutput.writeInt(2); + dataOutput.writeInt(0); + dataOutput.writeUTF(metadata.toTextBlock()); + + dataOutput.writeBoolean(false); + dataOutput.writeInt(1); + dataOutput.writeChar('a'); + dataOutput.writeInt(1); + dataOutput.writeInt(1); + dataOutput.writeUTF(serializedPatch); + dataOutput.writeInt(1); + + dataOutput.writeBoolean(false); + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeUTF(serializedPatch); + dataOutput.writeInt(2); + } + return gzip(rawOutputStream.toByteArray()); + } + + /** + * Compresses a binary payload using GZip. + * + * @param payload uncompressed bytes + * @return compressed bytes + * @throws IOException if compression fails + */ + private static byte[] gzip(final byte[] payload) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream)) { + gzipOutputStream.write(payload); + } + return outputStream.toByteArray(); + } + /** * Utility method that produces a small GZip-compressed byte array. * diff --git a/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java b/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java index b3a3e9a..fc8b9b7 100644 --- a/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java +++ b/src/test/java/org/egothor/stemmer/StemmerPatchTrieLoaderTest.java @@ -34,10 +34,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -58,6 +63,7 @@ import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import org.egothor.stemmer.StemmerPatchTrieLoader.Language; +import org.egothor.stemmer.trie.CompiledNode; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Tag; @@ -67,6 +73,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedStatic; /** * Professional test suite for {@link StemmerPatchTrieLoader}. @@ -386,6 +393,161 @@ final class StemmerPatchTrieLoaderTest { } } + /** + * Structural tests for direct compiled binary-loader delegation. + */ + @Nested + @DisplayName("Compiled binary routing") + @Tag("unit") + @Tag("io") + @Tag("persistence") + final class CompiledBinaryRoutingTests { + + /** + * Verifies that the path overload returns the direct compiled BinaryIO result + * without invoking deprecated String deserialization. + * + * @throws IOException if the loader unexpectedly reports an I/O failure + */ + @Test + @DisplayName("Path compiled loading must use direct compiled BinaryIO") + void shouldRouteCompiledPathLoadingDirectly() throws IOException { + final Path path = tempDir.resolve("compiled-routing-path.bin.gz"); + final FrequencyTrie compiledSentinel = compiledRoutingSentinel(); + final FrequencyTrie stringFallback = stringRoutingFallback(); + + try (MockedStatic binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) { + binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path)).thenReturn(compiledSentinel); + binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path)).thenReturn(stringFallback); + + final FrequencyTrie actual = StemmerPatchTrieLoader.loadBinaryCompiled(path); + + assertSame(compiledSentinel, actual, + "Path loading must return the trie materialized by the direct compiled reader."); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path), times(1)); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path), never()); + binaryIo.verifyNoMoreInteractions(); + } + } + + /** + * Verifies that the path-and-span overload preserves both arguments while + * delegating directly to compiled BinaryIO. + * + * @throws IOException if the loader unexpectedly reports an I/O failure + */ + @Test + @DisplayName("Path and span compiled loading must use direct compiled BinaryIO") + void shouldRouteCompiledPathAndSpanLoadingDirectly() throws IOException { + final Path path = tempDir.resolve("compiled-routing-path-span.bin.gz"); + final int maxExpandedIndex = 17; + final FrequencyTrie compiledSentinel = compiledRoutingSentinel(); + final FrequencyTrie stringFallback = stringRoutingFallback(); + + try (MockedStatic binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) { + binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex)) + .thenReturn(compiledSentinel); + binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex)).thenReturn(stringFallback); + + final FrequencyTrie actual = StemmerPatchTrieLoader.loadBinaryCompiled(path, + maxExpandedIndex); + + assertSame(compiledSentinel, actual, + "Path-and-span loading must return the trie materialized by the direct compiled reader."); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(path, maxExpandedIndex), times(1)); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex), never()); + binaryIo.verifyNoMoreInteractions(); + } + } + + /** + * Verifies that the file-name overload returns the direct compiled BinaryIO + * result without invoking deprecated String deserialization. + * + * @throws IOException if the loader unexpectedly reports an I/O failure + */ + @Test + @DisplayName("String compiled loading must use direct compiled BinaryIO") + void shouldRouteCompiledStringLoadingDirectly() throws IOException { + final String fileName = tempDir.resolve("compiled-routing-string.bin.gz").toString(); + final FrequencyTrie compiledSentinel = compiledRoutingSentinel(); + final FrequencyTrie stringFallback = stringRoutingFallback(); + + try (MockedStatic binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) { + binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName)).thenReturn(compiledSentinel); + binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName)).thenReturn(stringFallback); + + final FrequencyTrie actual = StemmerPatchTrieLoader + .loadBinaryCompiled(fileName); + + assertSame(compiledSentinel, actual, + "String loading must return the trie materialized by the direct compiled reader."); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName), times(1)); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName), never()); + binaryIo.verifyNoMoreInteractions(); + } + } + + /** + * Verifies that the file-name-and-span overload preserves both arguments + * while delegating directly to compiled BinaryIO. + * + * @throws IOException if the loader unexpectedly reports an I/O failure + */ + @Test + @DisplayName("String and span compiled loading must use direct compiled BinaryIO") + void shouldRouteCompiledStringAndSpanLoadingDirectly() throws IOException { + final String fileName = tempDir.resolve("compiled-routing-string-span.bin.gz").toString(); + final int maxExpandedIndex = 17; + final FrequencyTrie compiledSentinel = compiledRoutingSentinel(); + final FrequencyTrie stringFallback = stringRoutingFallback(); + + try (MockedStatic binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) { + binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex)) + .thenReturn(compiledSentinel); + binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex)) + .thenReturn(stringFallback); + + final FrequencyTrie actual = StemmerPatchTrieLoader + .loadBinaryCompiled(fileName, maxExpandedIndex); + + assertSame(compiledSentinel, actual, + "String-and-span loading must return the trie materialized by the direct compiled reader."); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(fileName, maxExpandedIndex), times(1)); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex), never()); + binaryIo.verifyNoMoreInteractions(); + } + } + + /** + * Verifies that the stream overload preserves stream identity while + * delegating directly to compiled BinaryIO. + * + * @throws IOException if the loader unexpectedly reports an I/O failure + */ + @Test + @DisplayName("InputStream compiled loading must use direct compiled BinaryIO") + void shouldRouteCompiledInputStreamLoadingDirectly() throws IOException { + final InputStream inputStream = new ByteArrayInputStream(new byte[] { 1, 2, 3 }); + final FrequencyTrie compiledSentinel = compiledRoutingSentinel(); + final FrequencyTrie stringFallback = stringRoutingFallback(); + + try (MockedStatic binaryIo = mockStatic(StemmerPatchTrieBinaryIO.class)) { + binaryIo.when(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream)).thenReturn(compiledSentinel); + binaryIo.when(() -> StemmerPatchTrieBinaryIO.read(inputStream)).thenReturn(stringFallback); + + final FrequencyTrie actual = StemmerPatchTrieLoader + .loadBinaryCompiled(inputStream); + + assertSame(compiledSentinel, actual, + "Stream loading must return the trie materialized by the direct compiled reader."); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.readCompiled(inputStream), times(1)); + binaryIo.verify(() -> StemmerPatchTrieBinaryIO.read(inputStream), never()); + binaryIo.verifyNoMoreInteractions(); + } + } + } + /** * Focused internal loader behavior tests. */ @@ -707,6 +869,54 @@ final class StemmerPatchTrieLoaderTest { } } + /** + * Verifies that public compiled binary loading materializes final command + * arrays directly and preserves shared table identity, counts, metadata, and + * canonical graph size. + * + * @throws IOException if writing or reading fails + */ + @Test + @DisplayName("Compiled binary loading should materialize one final shared command graph") + void shouldMaterializeOneFinalSharedCommandGraphFromBinary() throws IOException { + final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build(); + final String sharedPatch = encoder.encode("ab", "a"); + final String longerDeletionPatch = encoder.encode("xab", "x"); + final FrequencyTrie.Builder builder = new FrequencyTrie.Builder(String[]::new, + ReductionSettings + .withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS)); + builder.put("ab", new String(sharedPatch), 2); + builder.put("xab", longerDeletionPatch); + builder.put("cb", new String(sharedPatch), 3); + builder.put("ycb", new String(longerDeletionPatch)); + final FrequencyTrie sourceTrie = builder.build(); + final Path binaryFile = tempDir.resolve("direct-compiled-trie.bin.gz"); + StemmerPatchTrieLoader.saveBinary(sourceTrie, binaryFile); + + final FrequencyTrie compiledTrie = StemmerPatchTrieLoader + .loadBinaryCompiled(binaryFile); + final FrequencyTrie stringTrie = StemmerPatchTrieLoader.loadBinary(binaryFile); + final CompiledNode suffixBNode = compiledTrie.root().findChild('b'); + final CompiledNode abNode = suffixBNode.findChild('a'); + final CompiledNode cbNode = suffixBNode.findChild('c'); + + assertAll(() -> assertEquals(sourceTrie.metadata(), compiledTrie.metadata()), + () -> assertEquals(sourceTrie.size(), compiledTrie.size()), + () -> assertEquals(sourceTrie.getEntries("ab").get(0).count(), + compiledTrie.getEntries("ab").get(0).count()), + () -> assertEquals(sourceTrie.getEntries("cb").get(0).count(), + compiledTrie.getEntries("cb").get(0).count()), + () -> assertEquals(CompiledPatchCommand[].class, abNode.orderedValues().getClass()), + () -> assertSame(compiledTrie.get("ab"), compiledTrie.get("cb")), + () -> assertSame(abNode.orderedValues()[0], cbNode.orderedValues()[0]), + () -> assertEquals("a", compiledTrie.get("ab").apply("ab")), + () -> assertEquals("c", compiledTrie.get("cb").apply("cb")), + () -> assertEquals("x", compiledTrie.get("xab").apply("xab")), + () -> assertEquals("y", compiledTrie.get("ycb").apply("ycb")), + () -> assertInstanceOf(String.class, stringTrie.get("ab")), + () -> assertInstanceOf(CompiledPatchCommand.class, compiledTrie.get("ab"))); + } + /** * Verifies that binary load overloads with an explicit dense lookup span * preserve trie semantics while honoring the dense-layout override. @@ -728,9 +938,17 @@ final class StemmerPatchTrieLoaderTest { final FrequencyTrie fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile); final FrequencyTrie fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile, - FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX); + -1); final FrequencyTrie fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0); final FrequencyTrie fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0); + final FrequencyTrie compiledDefault = StemmerPatchTrieLoader + .loadBinaryCompiled(binaryFile); + final FrequencyTrie compiledDefaultByNegative = StemmerPatchTrieLoader + .loadBinaryCompiled(binaryFile, -1); + final FrequencyTrie compiledNoDense = StemmerPatchTrieLoader + .loadBinaryCompiled(binaryFile, 0); + final FrequencyTrie compiledStringNoDense = StemmerPatchTrieLoader + .loadBinaryCompiled(binaryFile.toString(), 0); assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying"); assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities", @@ -738,11 +956,27 @@ final class StemmerPatchTrieLoaderTest { assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying"); assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities", "studying"); + assertCompiledTrieSemanticsEqual(original, compiledDefault, "run", "running", "runner", "cities", + "studying"); + assertCompiledTrieSemanticsEqual(original, compiledDefaultByNegative, "run", "running", "runner", + "cities", "studying"); + assertCompiledTrieSemanticsEqual(original, compiledNoDense, "run", "running", "runner", "cities", + "studying"); + assertCompiledTrieSemanticsEqual(original, compiledStringNoDense, "run", "running", "runner", "cities", + "studying"); + assertTrue(compiledDefault.root().hasDenseLookup(), + "Default compiled loading should use the documented dense lookup span."); + assertTrue(compiledDefaultByNegative.root().hasDenseLookup(), + "Negative compiled override should use the documented default dense lookup span."); assertFalse(fromPathNoDense.root().hasDenseLookup(), "Zero span should disable dense lookup on the loaded root."); assertFalse(fromStringNoDense.root().hasDenseLookup(), "Zero span should disable dense lookup on the loaded root."); + assertFalse(compiledNoDense.root().hasDenseLookup(), + "Zero span should disable dense lookup on the directly compiled root."); + assertFalse(compiledStringNoDense.root().hasDenseLookup(), + "String-path zero span should disable dense lookup on the directly compiled root."); } /** @@ -990,6 +1224,36 @@ final class StemmerPatchTrieLoaderTest { return stems; } + /** + * Creates a compiled trie whose identity distinguishes the direct BinaryIO + * result from any graph produced through deprecated String loading. + * + * @return compiled trie sentinel + */ + private static FrequencyTrie compiledRoutingSentinel() { + final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run"); + final CompiledPatchCommand compiledPatch = CompiledPatchCommand.compile(serializedPatch, + WordTraversalDirection.BACKWARD); + return new FrequencyTrie.Builder(CompiledPatchCommand[]::new, + ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE)) + .put("running", compiledPatch) + .build(); + } + + /** + * Creates a valid String-valued trie that allows the deprecated two-stage route + * to execute rather than failing because of incomplete mock setup. + * + * @return valid String-valued fallback trie + */ + private static FrequencyTrie stringRoutingFallback() { + final String serializedPatch = PatchCommandEncoder.builder().build().encode("running", "run"); + return new FrequencyTrie.Builder(String[]::new, + ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE)) + .put("running", serializedPatch) + .build(); + } + /** * Verifies semantic equality of two tries for the supplied words by comparing * both their raw patch arrays and reconstructed stem sets.