From d2b974e1b8eba7a43b5aad42bb592dfe9207f045 Mon Sep 17 00:00:00 2001 From: Leo Galambos Date: Mon, 27 Jul 2026 20:16:48 +0200 Subject: [PATCH] perf(serialization): add value dictionary to trie stream v7 --- .../org/egothor/stemmer/FrequencyTrie.java | 150 +++++- .../egothor/stemmer/FrequencyTrieTest.java | 462 ++++++++++++++++++ 2 files changed, 599 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrie.java b/src/main/java/org/egothor/stemmer/FrequencyTrie.java index a59f90b..94b5bde 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrie.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrie.java @@ -179,7 +179,7 @@ public final class FrequencyTrie { /** * Binary format version. */ - private static final int STREAM_VERSION = 6; + private static final int STREAM_VERSION = 7; /** * Version where traversal-direction ordinal is persisted. @@ -206,6 +206,11 @@ public final class FrequencyTrie { */ private static final int ACCEPTING_NODE_VERSION = 6; + /** + * Version where distinct values are persisted once in a stream-local table. + */ + private static final int VALUE_TABLE_VERSION = 7; + /** * Argument name for lookup keys. */ @@ -748,6 +753,9 @@ public final class FrequencyTrie { final Map, Integer> nodeIds = new IdentityHashMap<>(); final List> orderedNodes = new ArrayList<>(); assignNodeIds(this.root, nodeIds, orderedNodes); + final Map valueIds = new LinkedHashMap<>(); + final List distinctValues = new ArrayList<>(); + collectDistinctValues(orderedNodes, valueIds, distinctValues); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.log(Level.FINE, "Writing compiled trie with {0} canonical nodes.", orderedNodes.size()); @@ -757,10 +765,11 @@ public final class FrequencyTrie { dataOutput.writeInt(STREAM_VERSION); dataOutput.writeInt(orderedNodes.size()); dataOutput.writeInt(nodeIds.get(this.root)); - writeMetadata(dataOutput, this.metadata); + writeMetadata(dataOutput, metadataForCurrentStream(this.metadata)); + writeValueTable(dataOutput, valueCodec, distinctValues); - for (CompiledNode node : orderedNodes) { - writeNode(dataOutput, valueCodec, node, nodeIds); + for (int nodeId = 0; nodeId < orderedNodes.size(); nodeId++) { + writeNode(dataOutput, orderedNodes.get(nodeId), nodeId, nodeIds, valueIds); } dataOutput.flush(); @@ -823,6 +832,24 @@ public final class FrequencyTrie { dataOutput.writeUTF(metadata.toTextBlock()); } + /** + * Creates metadata aligned with the stream version emitted by the current + * writer. + * + *

+ * The returned metadata preserves every semantic setting from the trie while + * reporting the current binary format version. The immutable metadata stored by + * the trie is not modified. + *

+ * + * @param metadata source trie metadata + * @return metadata aligned with {@link #STREAM_VERSION} + */ + private static TrieMetadata metadataForCurrentStream(final TrieMetadata metadata) { + return new TrieMetadata(STREAM_VERSION, metadata.traversalDirection(), metadata.reductionSettings(), + metadata.diacriticProcessingMode(), metadata.caseProcessingMode()); + } + /** * Returns the number of canonical compiled nodes reachable from the root. * @@ -864,16 +891,63 @@ public final class FrequencyTrie { } /** - * Writes one compiled node. + * Collects the deterministic equality-based value table for serialization. + * + *

+ * Nodes are visited in canonical node-identifier order and values are visited + * in their existing node-local order. The first occurrence according to + * {@link Object#equals(Object)} and {@link Object#hashCode()} assigns the table + * index. + *

+ * + * @param orderedNodes canonical nodes in identifier order + * @param valueIds destination mapping from values to table indexes + * @param distinctValues destination values in table-index order + * @param value type + */ + private static void collectDistinctValues(final List> orderedNodes, + final Map valueIds, final List distinctValues) { + for (CompiledNode node : orderedNodes) { + for (V value : node.orderedValues()) { + if (!valueIds.containsKey(value)) { + final int valueId = distinctValues.size(); + valueIds.put(value, valueId); + distinctValues.add(value); + } + } + } + } + + /** + * Writes every distinct value exactly once in table-index order. + * + * @param dataOutput output stream + * @param valueCodec codec responsible for value encoding + * @param distinctValues distinct values in deterministic table order + * @param value type + * @throws IOException if writing the table fails + */ + private static void writeValueTable(final DataOutputStream dataOutput, final ValueStreamCodec valueCodec, + final List distinctValues) throws IOException { + dataOutput.writeInt(distinctValues.size()); + for (V value : distinctValues) { + valueCodec.write(dataOutput, value); + } + } + + /** + * Writes one compiled node using stream-local value-table indexes. * * @param dataOutput output - * @param valueCodec value codec * @param node node to write + * @param nodeId canonical identifier of {@code node} * @param nodeIds node identifiers + * @param valueIds value-table indexes + * @param value type * @throws IOException if writing fails */ - private static void writeNode(final DataOutputStream dataOutput, final ValueStreamCodec valueCodec, - final CompiledNode node, final Map, Integer> nodeIds) throws IOException { + private static void writeNode(final DataOutputStream dataOutput, final CompiledNode node, final int nodeId, + final Map, Integer> nodeIds, final Map valueIds) throws IOException { dataOutput.writeBoolean(node.acceptsRemainingInput()); dataOutput.writeInt(node.edgeLabels().length); for (int index = 0; index < node.edgeLabels().length; index++) { @@ -887,7 +961,15 @@ public final class FrequencyTrie { dataOutput.writeInt(node.orderedValues().length); for (int index = 0; index < node.orderedValues().length; index++) { - valueCodec.write(dataOutput, node.orderedValues()[index]); + final V value = node.orderedValues()[index]; + final Integer valueId = valueIds.get(value); + if (valueId == null) { + final String valueContext = value == null ? "null" + : value.getClass().getName() + '[' + value + ']'; + throw new IOException("Missing value table index at canonical node " + nodeId + ", local value " + + index + ": " + valueContext); + } + dataOutput.writeInt(valueId); dataOutput.writeInt(node.orderedCounts()[index]); } } @@ -992,8 +1074,10 @@ public final class FrequencyTrie { } final TrieMetadata sourceMetadata = readMetadata(dataInput, version); + final V[] valueTable = version >= VALUE_TABLE_VERSION ? readValueTable(dataInput, arrayFactory, valueCodec) + : null; final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX; - final CompiledNode[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount, + final CompiledNode[] nodes = readNodes(dataInput, arrayFactory, valueCodec, valueTable, nodeCount, effectiveMaxExpandedIndex, version); final CompiledNode rootNode = nodes[rootNodeId]; @@ -1004,6 +1088,36 @@ public final class FrequencyTrie { return new FrequencyTrie<>(arrayFactory, rootNode, sourceMetadata); } + /** + * Reads the temporary stream-local value table. + * + *

+ * Each serialized value is decoded exactly once. The returned array is used + * only while materializing node value arrays and is not retained by the + * resulting trie. + *

+ * + * @param dataInput input stream + * @param arrayFactory typed-array factory + * @param valueCodec codec responsible for value decoding + * @param value type + * @return decoded values in table-index order + * @throws IOException if the count is negative or value decoding fails + */ + private static V[] readValueTable(final DataInputStream dataInput, final IntFunction arrayFactory, + final ValueStreamCodec valueCodec) throws IOException { + final int distinctValueCount = dataInput.readInt(); + if (distinctValueCount < 0) { + throw new IOException("Negative distinct value count: " + distinctValueCount); + } + + final V[] valueTable = arrayFactory.apply(distinctValueCount); + for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) { + valueTable[valueIndex] = valueCodec.read(dataInput); + } + return valueTable; + } + private static DataInputStream wrapInputStream(final InputStream inputStream) { return inputStream instanceof DataInputStream ? (DataInputStream) inputStream : new DataInputStream(inputStream); @@ -1067,8 +1181,8 @@ public final class FrequencyTrie { } private static CompiledNode[] readNodes(final DataInputStream dataInput, - final IntFunction arrayFactory, final ValueStreamCodec valueCodec, final int nodeCount, - final int maxExpandedIndex, final int version) throws IOException { + final IntFunction arrayFactory, final ValueStreamCodec valueCodec, final V[] valueTable, + final int nodeCount, final int maxExpandedIndex, final int version) throws IOException { final char[][] edgeLabelsByNode = new char[nodeCount][]; final int[][] childNodeIdsByNode = new int[nodeCount][]; @SuppressWarnings("unchecked") @@ -1111,7 +1225,17 @@ public final class FrequencyTrie { orderedCountsByNode[nodeIndex] = new int[valueCount]; for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { - orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput); + if (version >= VALUE_TABLE_VERSION) { + final int valueTableIndex = dataInput.readInt(); + if (valueTableIndex < 0 || valueTableIndex >= valueTable.length) { + throw new IOException("Invalid value table index at node " + nodeIndex + ", local value " + + valueIndex + ": " + valueTableIndex + "; table size is " + valueTable.length + + '.'); + } + orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex]; + } else { + orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput); + } orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt(); if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) { throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index " diff --git a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java index f0b8917..273105d 100644 --- a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java +++ b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java @@ -44,6 +44,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.EOFException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -94,6 +95,49 @@ class FrequencyTrieTest { } }; + /** + * Codec that records the number of encoded and decoded string values. + */ + private static final class CountingStringCodec implements FrequencyTrie.ValueStreamCodec { + + /** + * Number of completed write invocations. + */ + private int writeCount; + + /** + * Number of completed read invocations. + */ + private int readCount; + + /** + * Writes one string and records the invocation. + * + * @param dataOutput destination stream + * @param value value to encode + * @throws IOException if writing fails + */ + @Override + public void write(final DataOutputStream dataOutput, final String value) throws IOException { + dataOutput.writeUTF(value); + this.writeCount++; + } + + /** + * Reads one string and records the invocation. + * + * @param dataInput source stream + * @return decoded string + * @throws IOException if reading fails + */ + @Override + public String read(final DataInputStream dataInput) throws IOException { + final String value = dataInput.readUTF(); + this.readCount++; + return value; + } + } + /** * Creates a builder using the ranked get-all reduction mode. * @@ -104,6 +148,21 @@ class FrequencyTrieTest { ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS); } + /** + * Builds a backward trie whose repeated equal values remain on structurally + * distinct compiled nodes. + * + * @return trie containing two separate but equal shared-value inputs + */ + private static FrequencyTrie sharedValueTrie() { + final FrequencyTrie.Builder builder = rankedBuilder(); + builder.put("ab", new String("shared")); + builder.put("xab", "left"); + builder.put("cb", new String("shared")); + builder.put("ycb", "right"); + return builder.build(); + } + /** * Creates reduction settings with the internal uniform-subtree contraction * enabled. @@ -905,6 +964,7 @@ class FrequencyTrieTest { assertAll(() -> assertEquals(original.size(), restored.size()), () -> assertEquals(original.getFingerprint(), restored.getFingerprint()), + () -> assertEquals(original.metadata(), restored.metadata()), () -> assertEquals(original.get(""), restored.get("")), () -> assertArrayEquals(original.getAll(""), restored.getAll("")), () -> assertEquals(original.get("run"), restored.get("run")), @@ -924,6 +984,134 @@ class FrequencyTrieTest { () -> assertEquals(List.of(), restored.getEntries("missing"))); } + /** + * Verifies that the public current-format query reports stream version 7. + */ + @Test + @Tag("persistence") + @DisplayName("Current compiled trie format version is 7") + void currentCompiledTrieFormatVersionIsSeven() { + assertEquals(7, FrequencyTrie.currentFormatVersion()); + } + + /** + * Verifies the raw version 7 header, metadata, and deterministic value-table + * placement without relying on deserialization. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 stream writes header metadata and value table in order") + void versionSevenStreamWritesHeaderMetadataAndValueTableInOrder() throws IOException { + final FrequencyTrie currentTrie = sharedValueTrie(); + final TrieMetadata currentMetadata = currentTrie.metadata(); + final TrieMetadata historicalMetadata = new TrieMetadata(6, currentMetadata.traversalDirection(), + currentMetadata.reductionSettings(), currentMetadata.diacriticProcessingMode(), + currentMetadata.caseProcessingMode()); + final FrequencyTrie trie = FrequencyTrie.fromCompiled(String[]::new, currentTrie.root(), + historicalMetadata); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + trie.writeTo(outputStream, STRING_CODEC); + + final DataInputStream dataInput = new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray())); + final int magic = dataInput.readInt(); + final int version = dataInput.readInt(); + final int nodeCount = dataInput.readInt(); + final int rootNodeId = dataInput.readInt(); + final String metadataText = dataInput.readUTF(); + final int distinctValueCount = dataInput.readInt(); + final String firstValue = dataInput.readUTF(); + final String secondValue = dataInput.readUTF(); + final String thirdValue = dataInput.readUTF(); + + assertAll(() -> assertEquals(0x45475452, magic), + () -> assertEquals(7, version), + () -> assertTrue(nodeCount > 0), + () -> assertTrue(rootNodeId >= 0 && rootNodeId < nodeCount), + () -> assertTrue(metadataText.contains("\nformatVersion=7\n")), + () -> assertEquals(6, trie.metadata().formatVersion()), + () -> assertEquals(3, distinctValueCount), + () -> assertArrayEquals(new String[] { "shared", "left", "right" }, + new String[] { firstValue, secondValue, thirdValue })); + } + + /** + * Verifies that version 7 serialization deduplicates equal values represented + * by distinct Java objects. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 writer encodes each equality-distinct value once") + void versionSevenWriterEncodesEachEqualityDistinctValueOnce() throws IOException { + final FrequencyTrie trie = sharedValueTrie(); + final CountingStringCodec countingCodec = new CountingStringCodec(); + + trie.writeTo(new ByteArrayOutputStream(), countingCodec); + + assertEquals(3, countingCodec.writeCount); + } + + /** + * Verifies that version 7 deserialization invokes the value codec once per + * table entry rather than once per node-local slot. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader decodes each table value once") + void versionSevenReaderDecodesEachTableValueOnce() throws IOException { + final FrequencyTrie original = sharedValueTrie(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + original.writeTo(outputStream, STRING_CODEC); + final CountingStringCodec countingCodec = new CountingStringCodec(); + + final FrequencyTrie restored = FrequencyTrie.readFrom( + new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, countingCodec); + + assertAll(() -> assertEquals(3, countingCodec.readCount), + () -> assertEquals("shared", restored.get("ab")), + () -> assertEquals("shared", restored.get("cb")), + () -> assertEquals("left", restored.get("xab")), + () -> assertEquals("right", restored.get("ycb")), + () -> assertNull(restored.get("missing"))); + } + + /** + * Verifies that repeated version 7 table references become direct shared value + * references in the final compiled node arrays. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 materializes shared values directly in compiled nodes") + void versionSevenMaterializesSharedValuesDirectlyInCompiledNodes() throws IOException { + final FrequencyTrie original = sharedValueTrie(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + original.writeTo(outputStream, STRING_CODEC); + + final FrequencyTrie restored = FrequencyTrie.readFrom( + new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC); + final CompiledNode suffixBNode = restored.root().findChild('b'); + final CompiledNode abNode = suffixBNode.findChild('a'); + final CompiledNode cbNode = suffixBNode.findChild('c'); + final String abValue = abNode.orderedValues()[0]; + final String cbValue = cbNode.orderedValues()[0]; + + assertAll(() -> assertSame(restored.get("ab"), restored.get("cb")), + () -> assertSame(abValue, cbValue), + () -> assertSame(restored.get("ab"), abValue), + () -> assertEquals(String[].class, abNode.orderedValues().getClass()), + () -> assertEquals(String[].class, cbNode.orderedValues().getClass()), + () -> assertEquals("left", restored.get("xab")), + () -> assertEquals("right", restored.get("ycb")), + () -> assertNull(restored.get("missing"))); + } + /** * Verifies fingerprint stability and sensitivity to metadata and trie content. */ @@ -1689,6 +1877,156 @@ class FrequencyTrieTest { assertTrue(exception.getMessage().contains("Non-positive stored count")); } + /** + * Verifies that version 7 deserialization rejects a negative distinct-value + * count. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects negative value table size") + void versionSevenReaderRejectsNegativeValueTableSize() { + final byte[] bytes = createVersionSevenSerializedStream(dataOutput -> dataOutput.writeInt(-1), + dataOutput -> { + // The invalid table size is rejected before node decoding. + }); + + final IOException exception = assertThrows(IOException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + + assertEquals("Negative distinct value count: -1", exception.getMessage()); + } + + /** + * Verifies that version 7 deserialization rejects a negative value-table index + * with complete node-local context. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects negative value table index") + void versionSevenReaderRejectsNegativeValueTableIndex() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> writeVersionSevenValueNode(dataOutput, -1, 1)); + + final IOException exception = assertThrows(IOException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + + assertEquals("Invalid value table index at node 0, local value 0: -1; table size is 1.", + exception.getMessage()); + } + + /** + * Verifies that version 7 deserialization rejects an index equal to the + * value-table size. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects value table index equal to size") + void versionSevenReaderRejectsValueTableIndexEqualToSize() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> writeVersionSevenValueNode(dataOutput, 1, 1)); + + final IOException exception = assertThrows(IOException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + + assertEquals("Invalid value table index at node 0, local value 0: 1; table size is 1.", + exception.getMessage()); + } + + /** + * Verifies that version 7 deserialization rejects an index greater than the + * value-table size. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects value table index greater than size") + void versionSevenReaderRejectsValueTableIndexGreaterThanSize() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> writeVersionSevenValueNode(dataOutput, 2, 1)); + + final IOException exception = assertThrows(IOException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + + assertEquals("Invalid value table index at node 0, local value 0: 2; table size is 1.", + exception.getMessage()); + } + + /** + * Verifies that version 7 deserialization retains positive-count validation + * after resolving a valid table reference. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects zero occurrence count") + void versionSevenReaderRejectsZeroOccurrenceCount() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> writeVersionSevenValueNode(dataOutput, 0, 0)); + + final IOException exception = assertThrows(IOException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + + assertEquals("Non-positive stored count at node 0, value index 0: 0", exception.getMessage()); + } + + /** + * Verifies that a truncated version 7 value-table payload remains an + * {@link EOFException}. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects truncated value table") + void versionSevenReaderRejectsTruncatedValueTable() { + final byte[] bytes = createVersionSevenSerializedStream(dataOutput -> { + dataOutput.writeInt(1); + dataOutput.writeByte(0); + }, dataOutput -> { + // Value decoding fails before node decoding. + }); + + assertThrows(EOFException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + } + + /** + * Verifies that a truncated version 7 node-local table index remains an + * {@link EOFException}. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects truncated node-local value index") + void versionSevenReaderRejectsTruncatedNodeLocalValueIndex() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> { + dataOutput.writeBoolean(false); + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeShort(0); + }); + + assertThrows(EOFException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + } + + /** + * Verifies that a truncated version 7 occurrence count remains an + * {@link EOFException}. + */ + @Test + @Tag("persistence") + @DisplayName("Version 7 reader rejects truncated occurrence count") + void versionSevenReaderRejectsTruncatedOccurrenceCount() { + final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable, + dataOutput -> { + dataOutput.writeBoolean(false); + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeInt(0); + dataOutput.writeShort(1); + }); + + assertThrows(EOFException.class, + () -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC)); + } + /** * Verifies that legacy version 1 metadata uses compatibility defaults. */ @@ -1793,6 +2131,46 @@ class FrequencyTrieTest { () -> assertEquals(CaseProcessingMode.AS_IS, metadata.caseProcessingMode())); } + /** + * Verifies that historical text-metadata versions 5 and 6 retain inline value + * decoding without a value table. + * + * @throws IOException if test I/O fails unexpectedly + */ + @Test + @Tag("persistence") + @DisplayName("readFrom supports inline values in stream versions 5 and 6") + void readFromSupportsInlineValuesInStreamVersionsFiveAndSix() throws IOException { + for (int version = 5; version <= 6; version++) { + final int historicalVersion = version; + final TrieMetadata historicalMetadata = new TrieMetadata(historicalVersion, + WordTraversalDirection.BACKWARD, + ReductionSettings.withDefaults( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS), + DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT); + final byte[] bytes = createSerializedStream(0x45475452, historicalVersion, 1, 0, + dataOutput -> dataOutput.writeUTF(historicalMetadata.toTextBlock()), + new NodeWriter[] { dataOutput -> { + if (historicalVersion >= 6) { + dataOutput.writeBoolean(false); + } + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeUTF("inline"); + dataOutput.writeInt(2); + } }); + + final CountingStringCodec countingCodec = new CountingStringCodec(); + final FrequencyTrie trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, + countingCodec); + + assertAll(() -> assertEquals(historicalVersion, trie.metadata().formatVersion()), + () -> assertEquals("inline", trie.get("")), + () -> assertEquals(List.of(new ValueCount<>("inline", 2)), trie.getEntries("")), + () -> assertEquals(1, countingCodec.readCount)); + } + } + /** * Verifies that invalid legacy metadata ordinals are rejected by validation. */ @@ -1857,6 +2235,29 @@ class FrequencyTrieTest { */ private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount, final int rootNodeId, final MetadataWriter metadata, final NodeWriter[] nodes) { + return createSerializedStream(magic, version, nodeCount, rootNodeId, metadata, dataOutput -> { + if (version >= 7) { + dataOutput.writeInt(0); + } + }, nodes); + } + + /** + * Creates a synthetic serialized trie stream with metadata and value-table + * writer hooks. + * + * @param magic stream magic + * @param version stream version + * @param nodeCount declared node count + * @param rootNodeId declared root node identifier + * @param metadata version-specific metadata writer + * @param valueTable version-specific value-table writer + * @param nodes node body writers + * @return serialized bytes + */ + private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount, + final int rootNodeId, final MetadataWriter metadata, final ValueTableWriter valueTable, + final NodeWriter[] nodes) { try { final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); @@ -1866,6 +2267,7 @@ class FrequencyTrieTest { dataOutputStream.writeInt(nodeCount); dataOutputStream.writeInt(rootNodeId); metadata.write(dataOutputStream); + valueTable.write(dataOutputStream); for (NodeWriter node : nodes) { node.write(dataOutputStream); @@ -1878,6 +2280,51 @@ class FrequencyTrieTest { } } + /** + * Creates one synthetic version 7 stream containing a single declared node. + * + * @param valueTable value-table writer + * @param node node-body writer + * @return serialized bytes + */ + private static byte[] createVersionSevenSerializedStream(final ValueTableWriter valueTable, + final NodeWriter node) { + final TrieMetadata metadata = new TrieMetadata(7, WordTraversalDirection.BACKWARD, + ReductionSettings.withDefaults( + ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS), + DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT); + return createSerializedStream(0x45475452, 7, 1, 0, + dataOutput -> dataOutput.writeUTF(metadata.toTextBlock()), valueTable, new NodeWriter[] { node }); + } + + /** + * Writes one version 7 value table containing the string {@code value}. + * + * @param dataOutput output stream + * @throws IOException if writing fails + */ + private static void writeSingleValueTable(final DataOutputStream dataOutput) throws IOException { + dataOutput.writeInt(1); + dataOutput.writeUTF("value"); + } + + /** + * Writes one leaf node with a single version 7 value-table reference. + * + * @param dataOutput output stream + * @param valueTableIndex referenced table index + * @param occurrenceCount stored local occurrence count + * @throws IOException if writing fails + */ + private static void writeVersionSevenValueNode(final DataOutputStream dataOutput, final int valueTableIndex, + final int occurrenceCount) throws IOException { + dataOutput.writeBoolean(false); + dataOutput.writeInt(0); + dataOutput.writeInt(1); + dataOutput.writeInt(valueTableIndex); + dataOutput.writeInt(occurrenceCount); + } + /** * Writes one synthetic metadata block. */ @@ -1892,4 +2339,19 @@ class FrequencyTrieTest { */ void write(DataOutputStream dataOutput) throws IOException; } + + /** + * Writes the value-table section of a synthetic serialized trie stream. + */ + @FunctionalInterface + private interface ValueTableWriter { + + /** + * Writes one stream's version-specific value-table bytes. + * + * @param dataOutput output stream + * @throws IOException if writing fails + */ + void write(DataOutputStream dataOutput) throws IOException; + } }