Egothor/Radixor
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Add FrequencyTrie model fingerprints for EGOTHOR v4 analyzer identity

Browse Source 
The fingerprint covers trie metadata and the compiled node graph, exposes
a lowercase hex representation plus defensive raw bytes, and is stable
across equivalent trie builds and persistence round-trips.
This commit is contained in:
Leo Galambos
2026-05-24 19:54:29 +02:00
parent b945902f05
commit 464b580436
4 changed files with 225 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
src/main/java/org/egothor/stemmer/FrequencyTrie.java
View File

@@ -35,6 +35,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -93,6 +96,16 @@ public final class FrequencyTrie<V> {
*/
private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName());
/**
* Domain separator used by the trie fingerprint canonical input.
*/
private static final String FINGERPRINT_DOMAIN = "RADIXOR-FREQUENCY-TRIE-FINGERPRINT";
/**
* Version of the canonical fingerprint input format.
*/
private static final int FINGERPRINT_FORMAT_VERSION = 1;
/**
* Root node of the compiled read-only trie.
*/
@@ -103,6 +116,12 @@ public final class FrequencyTrie<V> {
*/
private final TrieMetadata metadata;
/**
* Canonical SHA-256 fingerprint bytes. The internal array is never exposed
* directly to callers.
*/
private final byte[] fingerprintBytes;
/**
* Cached traversal direction used for key lookup.
*/
@@ -233,6 +252,7 @@ public final class FrequencyTrie<V> {
final TrieMetadata metadata) {
this.root = Objects.requireNonNull(root, "root");
this.metadata = Objects.requireNonNull(metadata, "metadata");
this.fingerprintBytes = computeFingerprintBytes(root, metadata);
this.lookupTraversalDirection = metadata.traversalDirection();
this.lowercasesLookupKeys = metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
this.removeDiacritics = metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE;
@@ -504,6 +524,61 @@ public final class FrequencyTrie<V> {
return this.metadata;
}
/**
* Returns the deterministic SHA-256 fingerprint of this trie.
*
* <p>
* The fingerprint is a canonical model identity, not a Java object identity.
* It includes a fingerprint-domain marker, the fingerprint input format
* version, persisted metadata, and the complete compiled-node structure
* reachable from the root, including edges, child references, local values,
* and local counts.
* </p>
*
* <p>
* The returned value is stable across JVM runs for equivalent trie content and
* metadata. It does not include object identity, memory layout, runtime cache
* state, absolute file paths, timestamps, or other process-local state.
* </p>
*
* @return 64-character lowercase hexadecimal SHA-256 fingerprint
*/
public String getFingerprint() {
return toLowerHex(this.fingerprintBytes);
}
/**
* Returns a defensive copy of the raw SHA-256 fingerprint bytes.
*
* <p>
* The returned array has length {@code 32}. Mutating it does not affect this
* trie.
* </p>
*
* @return defensive copy of the 32-byte SHA-256 fingerprint
*/
public byte[] copyFingerprintBytes() {
return Arrays.copyOf(this.fingerprintBytes, this.fingerprintBytes.length);
}
private static <V> byte[] computeFingerprintBytes(final CompiledNode<V> root, final TrieMetadata metadata) {
final MessageDigest messageDigest = newSha256Digest();
updateUtf8(messageDigest, FINGERPRINT_DOMAIN);
updateInt(messageDigest, FINGERPRINT_FORMAT_VERSION);
updateUtf8(messageDigest, metadata.toTextBlock());
final Map<CompiledNode<V>, Integer> nodeIds = new IdentityHashMap<>();
final List<CompiledNode<V>> orderedNodes = new ArrayList<>();
assignNodeIds(root, nodeIds, orderedNodes);
updateInt(messageDigest, nodeIds.get(root));
updateInt(messageDigest, orderedNodes.size());
for (CompiledNode<V> node : orderedNodes) {
updateNodeFingerprint(messageDigest, node, nodeIds);
}
return messageDigest.digest();
}
/**
* Returns the root node mainly for diagnostics and tests within the package.
*
@@ -687,6 +762,64 @@ public final class FrequencyTrie<V> {
}
}
private static MessageDigest newSha256Digest() {
try {
return MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException exception) {
throw new IllegalStateException("SHA-256 digest is not available.", exception);
}
}
private static <V> void updateNodeFingerprint(final MessageDigest messageDigest, final CompiledNode<V> node,
final Map<CompiledNode<V>, Integer> nodeIds) {
final char[] edgeLabels = node.edgeLabels();
final CompiledNode<V>[] children = node.children();
final V[] values = node.orderedValues();
final int[] counts = node.orderedCounts();
updateInt(messageDigest, edgeLabels.length);
for (char edgeLabel : edgeLabels) {
updateInt(messageDigest, edgeLabel);
}
for (CompiledNode<V> child : children) {
final Integer childNodeId = nodeIds.get(child);
if (childNodeId == null) {
throw new IllegalStateException("Missing child node identifier during trie fingerprinting.");
}
updateInt(messageDigest, childNodeId.intValue());
}
updateInt(messageDigest, values.length);
for (V value : values) {
updateUtf8(messageDigest, String.valueOf(value));
}
for (int count : counts) {
updateInt(messageDigest, count);
}
}
private static void updateUtf8(final MessageDigest messageDigest, final String value) {
final byte[] encoded = value.getBytes(StandardCharsets.UTF_8);
updateInt(messageDigest, encoded.length);
messageDigest.update(encoded);
}
private static void updateInt(final MessageDigest messageDigest, final int value) {
messageDigest.update((byte) (value >>> 24));
messageDigest.update((byte) (value >>> 16));
messageDigest.update((byte) (value >>> 8));
messageDigest.update((byte) value);
}
private static String toLowerHex(final byte[] digest) {
final StringBuilder builder = new StringBuilder(digest.length * 2);
for (byte item : digest) {
builder.append(Character.forDigit((item >>> 4) & 0x0F, 16));
builder.append(Character.forDigit(item & 0x0F, 16));
}
return builder.toString();
}
/**
* Internal helper that materializes serialized trie data.
*

5
src/main/java/org/egothor/stemmer/trie/CompiledNode.java
View File

@@ -158,7 +158,7 @@ public final class CompiledNode<V> {
}
@SuppressWarnings("unchecked")
final CompiledNode<V>[] dense = (CompiledNode<V>[]) new CompiledNode[span + 1];
final CompiledNode<V>[] dense = new CompiledNode[span + 1];
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
}
@@ -347,7 +347,7 @@ public final class CompiledNode<V> {
/**
* Finds a child for the supplied edge character.
* <p>
*
* Lookup order is:
* <ol>
* <li>dense array index (if the label interval is compact enough),</li>
@@ -355,7 +355,6 @@ public final class CompiledNode<V> {
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
* <li>binary search over sorted labels.</li>
* </ol>
* </p>
*
* @param edge edge character
* @return child node, or {@code null} if absent

77
src/test/java/org/egothor/stemmer/FrequencyTrieTest.java
View File

@@ -873,6 +873,7 @@ class FrequencyTrieTest {
.readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
assertAll(() -> assertEquals(original.size(), restored.size()),
() -> assertEquals(original.getFingerprint(), restored.getFingerprint()),
() -> assertEquals(original.get(""), restored.get("")),
() -> assertArrayEquals(original.getAll(""), restored.getAll("")),
() -> assertEquals(original.get("run"), restored.get("run")),
@@ -892,6 +893,82 @@ class FrequencyTrieTest {
() -> assertEquals(List.of(), restored.getEntries("missing")));
}
/**
* Verifies fingerprint stability and sensitivity to metadata and trie content.
*/
@Test
@DisplayName("Fingerprint reflects metadata and compiled trie content")
void fingerprintReflectsMetadataAndCompiledTrieContent() {
final FrequencyTrie.Builder<String> baseBuilderA = rankedBuilder();
baseBuilderA.put("run", "verb", 3);
baseBuilderA.put("run", "noun", 1);
baseBuilderA.put("runner", "noun", 2);
final FrequencyTrie<String> trieA = baseBuilderA.build();
final FrequencyTrie.Builder<String> baseBuilderB = rankedBuilder();
baseBuilderB.put("run", "verb", 3);
baseBuilderB.put("run", "noun", 1);
baseBuilderB.put("runner", "noun", 2);
final FrequencyTrie<String> trieB = baseBuilderB.build();
final FrequencyTrie.Builder<String> reorderedBuilder = rankedBuilder();
reorderedBuilder.put("runner", "noun", 2);
reorderedBuilder.put("run", "noun", 1);
reorderedBuilder.put("run", "verb", 3);
final FrequencyTrie<String> reorderedTrie = reorderedBuilder.build();
final FrequencyTrie.Builder<String> differentContentBuilder = rankedBuilder();
differentContentBuilder.put("run", "verb", 3);
differentContentBuilder.put("run", "noun", 2);
differentContentBuilder.put("runner", "noun", 2);
final FrequencyTrie<String> differentContentTrie = differentContentBuilder.build();
final FrequencyTrie.Builder<String> differentMetadataBuilder = new FrequencyTrie.Builder<>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
WordTraversalDirection.FORWARD, CaseProcessingMode.AS_IS);
differentMetadataBuilder.put("run", "verb", 3);
differentMetadataBuilder.put("run", "noun", 1);
differentMetadataBuilder.put("runner", "noun", 2);
final FrequencyTrie<String> differentMetadataTrie = differentMetadataBuilder.build();
final String fingerprintA = trieA.getFingerprint();
final String fingerprintB = trieB.getFingerprint();
final String reorderedFingerprint = reorderedTrie.getFingerprint();
final String differentContentFingerprint = differentContentTrie.getFingerprint();
final String differentMetadataFingerprint = differentMetadataTrie.getFingerprint();
final byte[] fingerprintBytes = trieA.copyFingerprintBytes();
final byte[] secondFingerprintBytes = trieA.copyFingerprintBytes();
fingerprintBytes[0] = (byte) (fingerprintBytes[0] ^ 0x7F);
assertAll(() -> assertEquals(fingerprintA, fingerprintB),
() -> assertEquals(fingerprintA, reorderedFingerprint),
() -> assertEquals(fingerprintA, trieA.getFingerprint()),
() -> assertFalse(fingerprintA.isBlank()),
() -> assertLowercaseSha256Hex(fingerprintA),
() -> assertEquals(fingerprintA, toLowerHex(secondFingerprintBytes)),
() -> assertArrayEquals(secondFingerprintBytes, trieA.copyFingerprintBytes()),
() -> assertFalse(fingerprintA.equals(differentContentFingerprint)),
() -> assertFalse(fingerprintA.equals(differentMetadataFingerprint)));
}
private static void assertLowercaseSha256Hex(final String fingerprint) {
assertEquals(64, fingerprint.length());
for (int index = 0; index < fingerprint.length(); index++) {
final char character = fingerprint.charAt(index);
final boolean digit = character >= '0' && character <= '9';
final boolean lowercaseHex = character >= 'a' && character <= 'f';
assertTrue(digit || lowercaseHex, "Invalid fingerprint character at index " + index + '.');
}
}
private static String toLowerHex(final byte[] bytes) {
final StringBuilder builder = new StringBuilder(bytes.length * 2);
for (byte item : bytes) {
builder.append(Character.forDigit((item >>> 4) & 0x0F, 16));
builder.append(Character.forDigit(item & 0x0F, 16));
}
return builder.toString();
}
/**
* Verifies that persistence methods reject {@code null} arguments.
*