Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(serialization): add value dictionary to trie stream v7

Browse Source
This commit is contained in:
Leo Galambos
2026-07-27 20:16:48 +02:00
parent 75d31aeda5
commit d2b974e1b8
2 changed files with 599 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -179,7 +179,7 @@ public final class FrequencyTrie<V> {
/**
* Binary format version.
*/
private static final int STREAM_VERSION = 6;
private static final int STREAM_VERSION = 7;
/**
* Version where traversal-direction ordinal is persisted.
@@ -206,6 +206,11 @@ public final class FrequencyTrie<V> {
*/
private static final int ACCEPTING_NODE_VERSION = 6;
/**
* Version where distinct values are persisted once in a stream-local table.
*/
private static final int VALUE_TABLE_VERSION = 7;
/**
* Argument name for lookup keys.
*/
@@ -748,6 +753,9 @@ public final class FrequencyTrie<V> {
final Map<CompiledNode<V>, Integer> nodeIds = new IdentityHashMap<>();
final List<CompiledNode<V>> orderedNodes = new ArrayList<>();
assignNodeIds(this.root, nodeIds, orderedNodes);
final Map<V, Integer> valueIds = new LinkedHashMap<>();
final List<V> distinctValues = new ArrayList<>();
collectDistinctValues(orderedNodes, valueIds, distinctValues);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log(Level.FINE, "Writing compiled trie with {0} canonical nodes.", orderedNodes.size());
@@ -757,10 +765,11 @@ public final class FrequencyTrie<V> {
dataOutput.writeInt(STREAM_VERSION);
dataOutput.writeInt(orderedNodes.size());
dataOutput.writeInt(nodeIds.get(this.root));
writeMetadata(dataOutput, this.metadata);
writeMetadata(dataOutput, metadataForCurrentStream(this.metadata));
writeValueTable(dataOutput, valueCodec, distinctValues);
for (CompiledNode<V> node : orderedNodes) {
writeNode(dataOutput, valueCodec, node, nodeIds);
for (int nodeId = 0; nodeId < orderedNodes.size(); nodeId++) {
writeNode(dataOutput, orderedNodes.get(nodeId), nodeId, nodeIds, valueIds);
}
dataOutput.flush();
@@ -823,6 +832,24 @@ public final class FrequencyTrie<V> {
dataOutput.writeUTF(metadata.toTextBlock());
}
/**
* Creates metadata aligned with the stream version emitted by the current
* writer.
*
* <p>
* The returned metadata preserves every semantic setting from the trie while
* reporting the current binary format version. The immutable metadata stored by
* the trie is not modified.
* </p>
*
* @param metadata source trie metadata
* @return metadata aligned with {@link #STREAM_VERSION}
*/
private static TrieMetadata metadataForCurrentStream(final TrieMetadata metadata) {
return new TrieMetadata(STREAM_VERSION, metadata.traversalDirection(), metadata.reductionSettings(),
metadata.diacriticProcessingMode(), metadata.caseProcessingMode());
}
/**
* Returns the number of canonical compiled nodes reachable from the root.
*
@@ -864,16 +891,63 @@ public final class FrequencyTrie<V> {
}
/**
* Writes one compiled node.
* Collects the deterministic equality-based value table for serialization.
*
* <p>
* Nodes are visited in canonical node-identifier order and values are visited
* in their existing node-local order. The first occurrence according to
* {@link Object#equals(Object)} and {@link Object#hashCode()} assigns the table
* index.
* </p>
*
* @param orderedNodes canonical nodes in identifier order
* @param valueIds destination mapping from values to table indexes
* @param distinctValues destination values in table-index order
* @param <V> value type
*/
private static <V> void collectDistinctValues(final List<CompiledNode<V>> orderedNodes,
final Map<V, Integer> valueIds, final List<V> distinctValues) {
for (CompiledNode<V> node : orderedNodes) {
for (V value : node.orderedValues()) {
if (!valueIds.containsKey(value)) {
final int valueId = distinctValues.size();
valueIds.put(value, valueId);
distinctValues.add(value);
}
}
}
}
/**
* Writes every distinct value exactly once in table-index order.
*
* @param dataOutput output stream
* @param valueCodec codec responsible for value encoding
* @param distinctValues distinct values in deterministic table order
* @param <V> value type
* @throws IOException if writing the table fails
*/
private static <V> void writeValueTable(final DataOutputStream dataOutput, final ValueStreamCodec<V> valueCodec,
final List<V> distinctValues) throws IOException {
dataOutput.writeInt(distinctValues.size());
for (V value : distinctValues) {
valueCodec.write(dataOutput, value);
}
}
/**
* Writes one compiled node using stream-local value-table indexes.
*
* @param dataOutput output
* @param valueCodec value codec
* @param node node to write
* @param nodeId canonical identifier of {@code node}
* @param nodeIds node identifiers
* @param valueIds value-table indexes
* @param <V> value type
* @throws IOException if writing fails
*/
private static <V> void writeNode(final DataOutputStream dataOutput, final ValueStreamCodec<V> valueCodec,
final CompiledNode<V> node, final Map<CompiledNode<V>, Integer> nodeIds) throws IOException {
private static <V> void writeNode(final DataOutputStream dataOutput, final CompiledNode<V> node, final int nodeId,
final Map<CompiledNode<V>, Integer> nodeIds, final Map<V, Integer> valueIds) throws IOException {
dataOutput.writeBoolean(node.acceptsRemainingInput());
dataOutput.writeInt(node.edgeLabels().length);
for (int index = 0; index < node.edgeLabels().length; index++) {
@@ -887,7 +961,15 @@ public final class FrequencyTrie<V> {
dataOutput.writeInt(node.orderedValues().length);
for (int index = 0; index < node.orderedValues().length; index++) {
valueCodec.write(dataOutput, node.orderedValues()[index]);
final V value = node.orderedValues()[index];
final Integer valueId = valueIds.get(value);
if (valueId == null) {
final String valueContext = value == null ? "null"
: value.getClass().getName() + '[' + value + ']';
throw new IOException("Missing value table index at canonical node " + nodeId + ", local value "
+ index + ": " + valueContext);
}
dataOutput.writeInt(valueId);
dataOutput.writeInt(node.orderedCounts()[index]);
}
}
@@ -992,8 +1074,10 @@ public final class FrequencyTrie<V> {
}
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
final V[] valueTable = version >= VALUE_TABLE_VERSION ? readValueTable(dataInput, arrayFactory, valueCodec)
: null;
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount,
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, valueTable, nodeCount,
effectiveMaxExpandedIndex, version);
final CompiledNode<V> rootNode = nodes[rootNodeId];
@@ -1004,6 +1088,36 @@ public final class FrequencyTrie<V> {
return new FrequencyTrie<>(arrayFactory, rootNode, sourceMetadata);
}
/**
* Reads the temporary stream-local value table.
*
* <p>
* Each serialized value is decoded exactly once. The returned array is used
* only while materializing node value arrays and is not retained by the
* resulting trie.
* </p>
*
* @param dataInput input stream
* @param arrayFactory typed-array factory
* @param valueCodec codec responsible for value decoding
* @param <V> value type
* @return decoded values in table-index order
* @throws IOException if the count is negative or value decoding fails
*/
private static <V> V[] readValueTable(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
final ValueStreamCodec<V> valueCodec) throws IOException {
final int distinctValueCount = dataInput.readInt();
if (distinctValueCount < 0) {
throw new IOException("Negative distinct value count: " + distinctValueCount);
}
final V[] valueTable = arrayFactory.apply(distinctValueCount);
for (int valueIndex = 0; valueIndex < distinctValueCount; valueIndex++) {
valueTable[valueIndex] = valueCodec.read(dataInput);
}
return valueTable;
}
private static DataInputStream wrapInputStream(final InputStream inputStream) {
return inputStream instanceof DataInputStream ? (DataInputStream) inputStream
: new DataInputStream(inputStream);
@@ -1067,8 +1181,8 @@ public final class FrequencyTrie<V> {
}
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final int nodeCount,
final int maxExpandedIndex, final int version) throws IOException {
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final V[] valueTable,
final int nodeCount, final int maxExpandedIndex, final int version) throws IOException {
final char[][] edgeLabelsByNode = new char[nodeCount][];
final int[][] childNodeIdsByNode = new int[nodeCount][];
@SuppressWarnings("unchecked")
@@ -1111,7 +1225,17 @@ public final class FrequencyTrie<V> {
orderedCountsByNode[nodeIndex] = new int[valueCount];
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
if (version >= VALUE_TABLE_VERSION) {
final int valueTableIndex = dataInput.readInt();
if (valueTableIndex < 0 || valueTableIndex >= valueTable.length) {
throw new IOException("Invalid value table index at node " + nodeIndex + ", local value "
+ valueIndex + ": " + valueTableIndex + "; table size is " + valueTable.length
+ '.');
}
orderedValuesByNode[nodeIndex][valueIndex] = valueTable[valueTableIndex];
} else {
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
}
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index "

462
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -44,6 +44,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -94,6 +95,49 @@ class FrequencyTrieTest {
}
};
/**
* Codec that records the number of encoded and decoded string values.
*/
private static final class CountingStringCodec implements FrequencyTrie.ValueStreamCodec<String> {
/**
* Number of completed write invocations.
*/
private int writeCount;
/**
* Number of completed read invocations.
*/
private int readCount;
/**
* Writes one string and records the invocation.
*
* @param dataOutput destination stream
* @param value value to encode
* @throws IOException if writing fails
*/
@Override
public void write(final DataOutputStream dataOutput, final String value) throws IOException {
dataOutput.writeUTF(value);
this.writeCount++;
}
/**
* Reads one string and records the invocation.
*
* @param dataInput source stream
* @return decoded string
* @throws IOException if reading fails
*/
@Override
public String read(final DataInputStream dataInput) throws IOException {
final String value = dataInput.readUTF();
this.readCount++;
return value;
}
}
/**
* Creates a builder using the ranked get-all reduction mode.
*
@@ -104,6 +148,21 @@ class FrequencyTrieTest {
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
}
/**
* Builds a backward trie whose repeated equal values remain on structurally
* distinct compiled nodes.
*
* @return trie containing two separate but equal shared-value inputs
*/
private static FrequencyTrie<String> sharedValueTrie() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("ab", new String("shared"));
builder.put("xab", "left");
builder.put("cb", new String("shared"));
builder.put("ycb", "right");
return builder.build();
}
/**
* Creates reduction settings with the internal uniform-subtree contraction
* enabled.
@@ -905,6 +964,7 @@ class FrequencyTrieTest {
assertAll(() -> assertEquals(original.size(), restored.size()),
() -> assertEquals(original.getFingerprint(), restored.getFingerprint()),
() -> assertEquals(original.metadata(), restored.metadata()),
() -> assertEquals(original.get(""), restored.get("")),
() -> assertArrayEquals(original.getAll(""), restored.getAll("")),
() -> assertEquals(original.get("run"), restored.get("run")),
@@ -924,6 +984,134 @@ class FrequencyTrieTest {
() -> assertEquals(List.of(), restored.getEntries("missing")));
}
/**
* Verifies that the public current-format query reports stream version 7.
*/
@Test
@Tag("persistence")
@DisplayName("Current compiled trie format version is 7")
void currentCompiledTrieFormatVersionIsSeven() {
assertEquals(7, FrequencyTrie.currentFormatVersion());
}
/**
* Verifies the raw version 7 header, metadata, and deterministic value-table
* placement without relying on deserialization.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 stream writes header metadata and value table in order")
void versionSevenStreamWritesHeaderMetadataAndValueTableInOrder() throws IOException {
final FrequencyTrie<String> currentTrie = sharedValueTrie();
final TrieMetadata currentMetadata = currentTrie.metadata();
final TrieMetadata historicalMetadata = new TrieMetadata(6, currentMetadata.traversalDirection(),
currentMetadata.reductionSettings(), currentMetadata.diacriticProcessingMode(),
currentMetadata.caseProcessingMode());
final FrequencyTrie<String> trie = FrequencyTrie.fromCompiled(String[]::new, currentTrie.root(),
historicalMetadata);
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
trie.writeTo(outputStream, STRING_CODEC);
final DataInputStream dataInput = new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray()));
final int magic = dataInput.readInt();
final int version = dataInput.readInt();
final int nodeCount = dataInput.readInt();
final int rootNodeId = dataInput.readInt();
final String metadataText = dataInput.readUTF();
final int distinctValueCount = dataInput.readInt();
final String firstValue = dataInput.readUTF();
final String secondValue = dataInput.readUTF();
final String thirdValue = dataInput.readUTF();
assertAll(() -> assertEquals(0x45475452, magic),
() -> assertEquals(7, version),
() -> assertTrue(nodeCount > 0),
() -> assertTrue(rootNodeId >= 0 && rootNodeId < nodeCount),
() -> assertTrue(metadataText.contains("\nformatVersion=7\n")),
() -> assertEquals(6, trie.metadata().formatVersion()),
() -> assertEquals(3, distinctValueCount),
() -> assertArrayEquals(new String[] { "shared", "left", "right" },
new String[] { firstValue, secondValue, thirdValue }));
}
/**
* Verifies that version 7 serialization deduplicates equal values represented
* by distinct Java objects.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 writer encodes each equality-distinct value once")
void versionSevenWriterEncodesEachEqualityDistinctValueOnce() throws IOException {
final FrequencyTrie<String> trie = sharedValueTrie();
final CountingStringCodec countingCodec = new CountingStringCodec();
trie.writeTo(new ByteArrayOutputStream(), countingCodec);
assertEquals(3, countingCodec.writeCount);
}
/**
* Verifies that version 7 deserialization invokes the value codec once per
* table entry rather than once per node-local slot.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader decodes each table value once")
void versionSevenReaderDecodesEachTableValueOnce() throws IOException {
final FrequencyTrie<String> original = sharedValueTrie();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
original.writeTo(outputStream, STRING_CODEC);
final CountingStringCodec countingCodec = new CountingStringCodec();
final FrequencyTrie<String> restored = FrequencyTrie.readFrom(
new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, countingCodec);
assertAll(() -> assertEquals(3, countingCodec.readCount),
() -> assertEquals("shared", restored.get("ab")),
() -> assertEquals("shared", restored.get("cb")),
() -> assertEquals("left", restored.get("xab")),
() -> assertEquals("right", restored.get("ycb")),
() -> assertNull(restored.get("missing")));
}
/**
* Verifies that repeated version 7 table references become direct shared value
* references in the final compiled node arrays.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 materializes shared values directly in compiled nodes")
void versionSevenMaterializesSharedValuesDirectlyInCompiledNodes() throws IOException {
final FrequencyTrie<String> original = sharedValueTrie();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
original.writeTo(outputStream, STRING_CODEC);
final FrequencyTrie<String> restored = FrequencyTrie.readFrom(
new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
final CompiledNode<String> suffixBNode = restored.root().findChild('b');
final CompiledNode<String> abNode = suffixBNode.findChild('a');
final CompiledNode<String> cbNode = suffixBNode.findChild('c');
final String abValue = abNode.orderedValues()[0];
final String cbValue = cbNode.orderedValues()[0];
assertAll(() -> assertSame(restored.get("ab"), restored.get("cb")),
() -> assertSame(abValue, cbValue),
() -> assertSame(restored.get("ab"), abValue),
() -> assertEquals(String[].class, abNode.orderedValues().getClass()),
() -> assertEquals(String[].class, cbNode.orderedValues().getClass()),
() -> assertEquals("left", restored.get("xab")),
() -> assertEquals("right", restored.get("ycb")),
() -> assertNull(restored.get("missing")));
}
/**
* Verifies fingerprint stability and sensitivity to metadata and trie content.
*/
@@ -1689,6 +1877,156 @@ class FrequencyTrieTest {
assertTrue(exception.getMessage().contains("Non-positive stored count"));
}
/**
* Verifies that version 7 deserialization rejects a negative distinct-value
* count.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects negative value table size")
void versionSevenReaderRejectsNegativeValueTableSize() {
final byte[] bytes = createVersionSevenSerializedStream(dataOutput -> dataOutput.writeInt(-1),
dataOutput -> {
// The invalid table size is rejected before node decoding.
});
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertEquals("Negative distinct value count: -1", exception.getMessage());
}
/**
* Verifies that version 7 deserialization rejects a negative value-table index
* with complete node-local context.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects negative value table index")
void versionSevenReaderRejectsNegativeValueTableIndex() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> writeVersionSevenValueNode(dataOutput, -1, 1));
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertEquals("Invalid value table index at node 0, local value 0: -1; table size is 1.",
exception.getMessage());
}
/**
* Verifies that version 7 deserialization rejects an index equal to the
* value-table size.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects value table index equal to size")
void versionSevenReaderRejectsValueTableIndexEqualToSize() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> writeVersionSevenValueNode(dataOutput, 1, 1));
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertEquals("Invalid value table index at node 0, local value 0: 1; table size is 1.",
exception.getMessage());
}
/**
* Verifies that version 7 deserialization rejects an index greater than the
* value-table size.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects value table index greater than size")
void versionSevenReaderRejectsValueTableIndexGreaterThanSize() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> writeVersionSevenValueNode(dataOutput, 2, 1));
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertEquals("Invalid value table index at node 0, local value 0: 2; table size is 1.",
exception.getMessage());
}
/**
* Verifies that version 7 deserialization retains positive-count validation
* after resolving a valid table reference.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects zero occurrence count")
void versionSevenReaderRejectsZeroOccurrenceCount() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> writeVersionSevenValueNode(dataOutput, 0, 0));
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertEquals("Non-positive stored count at node 0, value index 0: 0", exception.getMessage());
}
/**
* Verifies that a truncated version 7 value-table payload remains an
* {@link EOFException}.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects truncated value table")
void versionSevenReaderRejectsTruncatedValueTable() {
final byte[] bytes = createVersionSevenSerializedStream(dataOutput -> {
dataOutput.writeInt(1);
dataOutput.writeByte(0);
}, dataOutput -> {
// Value decoding fails before node decoding.
});
assertThrows(EOFException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
}
/**
* Verifies that a truncated version 7 node-local table index remains an
* {@link EOFException}.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects truncated node-local value index")
void versionSevenReaderRejectsTruncatedNodeLocalValueIndex() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> {
dataOutput.writeBoolean(false);
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeShort(0);
});
assertThrows(EOFException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
}
/**
* Verifies that a truncated version 7 occurrence count remains an
* {@link EOFException}.
*/
@Test
@Tag("persistence")
@DisplayName("Version 7 reader rejects truncated occurrence count")
void versionSevenReaderRejectsTruncatedOccurrenceCount() {
final byte[] bytes = createVersionSevenSerializedStream(FrequencyTrieTest::writeSingleValueTable,
dataOutput -> {
dataOutput.writeBoolean(false);
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeInt(0);
dataOutput.writeShort(1);
});
assertThrows(EOFException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
}
/**
* Verifies that legacy version 1 metadata uses compatibility defaults.
*/
@@ -1793,6 +2131,46 @@ class FrequencyTrieTest {
() -> assertEquals(CaseProcessingMode.AS_IS, metadata.caseProcessingMode()));
}
/**
* Verifies that historical text-metadata versions 5 and 6 retain inline value
* decoding without a value table.
*
* @throws IOException if test I/O fails unexpectedly
*/
@Test
@Tag("persistence")
@DisplayName("readFrom supports inline values in stream versions 5 and 6")
void readFromSupportsInlineValuesInStreamVersionsFiveAndSix() throws IOException {
for (int version = 5; version <= 6; version++) {
final int historicalVersion = version;
final TrieMetadata historicalMetadata = new TrieMetadata(historicalVersion,
WordTraversalDirection.BACKWARD,
ReductionSettings.withDefaults(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
final byte[] bytes = createSerializedStream(0x45475452, historicalVersion, 1, 0,
dataOutput -> dataOutput.writeUTF(historicalMetadata.toTextBlock()),
new NodeWriter[] { dataOutput -> {
if (historicalVersion >= 6) {
dataOutput.writeBoolean(false);
}
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeUTF("inline");
dataOutput.writeInt(2);
} });
final CountingStringCodec countingCodec = new CountingStringCodec();
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new,
countingCodec);
assertAll(() -> assertEquals(historicalVersion, trie.metadata().formatVersion()),
() -> assertEquals("inline", trie.get("")),
() -> assertEquals(List.of(new ValueCount<>("inline", 2)), trie.getEntries("")),
() -> assertEquals(1, countingCodec.readCount));
}
}
/**
* Verifies that invalid legacy metadata ordinals are rejected by validation.
*/
@@ -1857,6 +2235,29 @@ class FrequencyTrieTest {
*/
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
final int rootNodeId, final MetadataWriter metadata, final NodeWriter[] nodes) {
return createSerializedStream(magic, version, nodeCount, rootNodeId, metadata, dataOutput -> {
if (version >= 7) {
dataOutput.writeInt(0);
}
}, nodes);
}
/**
* Creates a synthetic serialized trie stream with metadata and value-table
* writer hooks.
*
* @param magic stream magic
* @param version stream version
* @param nodeCount declared node count
* @param rootNodeId declared root node identifier
* @param metadata version-specific metadata writer
* @param valueTable version-specific value-table writer
* @param nodes node body writers
* @return serialized bytes
*/
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
final int rootNodeId, final MetadataWriter metadata, final ValueTableWriter valueTable,
final NodeWriter[] nodes) {
try {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
@@ -1866,6 +2267,7 @@ class FrequencyTrieTest {
dataOutputStream.writeInt(nodeCount);
dataOutputStream.writeInt(rootNodeId);
metadata.write(dataOutputStream);
valueTable.write(dataOutputStream);
for (NodeWriter node : nodes) {
node.write(dataOutputStream);
@@ -1878,6 +2280,51 @@ class FrequencyTrieTest {
}
}
/**
* Creates one synthetic version 7 stream containing a single declared node.
*
* @param valueTable value-table writer
* @param node node-body writer
* @return serialized bytes
*/
private static byte[] createVersionSevenSerializedStream(final ValueTableWriter valueTable,
final NodeWriter node) {
final TrieMetadata metadata = new TrieMetadata(7, WordTraversalDirection.BACKWARD,
ReductionSettings.withDefaults(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
return createSerializedStream(0x45475452, 7, 1, 0,
dataOutput -> dataOutput.writeUTF(metadata.toTextBlock()), valueTable, new NodeWriter[] { node });
}
/**
* Writes one version 7 value table containing the string {@code value}.
*
* @param dataOutput output stream
* @throws IOException if writing fails
*/
private static void writeSingleValueTable(final DataOutputStream dataOutput) throws IOException {
dataOutput.writeInt(1);
dataOutput.writeUTF("value");
}
/**
* Writes one leaf node with a single version 7 value-table reference.
*
* @param dataOutput output stream
* @param valueTableIndex referenced table index
* @param occurrenceCount stored local occurrence count
* @throws IOException if writing fails
*/
private static void writeVersionSevenValueNode(final DataOutputStream dataOutput, final int valueTableIndex,
final int occurrenceCount) throws IOException {
dataOutput.writeBoolean(false);
dataOutput.writeInt(0);
dataOutput.writeInt(1);
dataOutput.writeInt(valueTableIndex);
dataOutput.writeInt(occurrenceCount);
}
/**
* Writes one synthetic metadata block.
*/
@@ -1892,4 +2339,19 @@ class FrequencyTrieTest {
*/
void write(DataOutputStream dataOutput) throws IOException;
}
/**
* Writes the value-table section of a synthetic serialized trie stream.
*/
@FunctionalInterface
private interface ValueTableWriter {
/**
* Writes one stream's version-specific value-table bytes.
*
* @param dataOutput output stream
* @throws IOException if writing fails
*/
void write(DataOutputStream dataOutput) throws IOException;
}
}