feat: Add FrequencyTrie model fingerprints for EGOTHOR v4 analyzer identity
The fingerprint covers trie metadata and the compiled node graph, exposes a lowercase hex representation plus defensive raw bytes, and is stable across equivalent trie builds and persistence round-trips.
This commit is contained in:
27
.project
27
.project
@@ -2,22 +2,21 @@
|
|||||||
<projectDescription>
|
<projectDescription>
|
||||||
<name>Radixor</name>
|
<name>Radixor</name>
|
||||||
<comment></comment>
|
<comment></comment>
|
||||||
<projects>
|
<projects/>
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
<natures>
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||||
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
||||||
</natures>
|
</natures>
|
||||||
|
<buildSpec>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||||
|
<arguments/>
|
||||||
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||||
|
<arguments/>
|
||||||
|
</buildCommand>
|
||||||
|
</buildSpec>
|
||||||
|
<linkedResources/>
|
||||||
|
<filteredResources/>
|
||||||
</projectDescription>
|
</projectDescription>
|
||||||
|
|||||||
@@ -35,6 +35,9 @@ import java.io.DataOutputStream;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
@@ -86,6 +89,7 @@ import org.egothor.stemmer.trie.ReductionSignature;
|
|||||||
*
|
*
|
||||||
* @param <V> value type
|
* @param <V> value type
|
||||||
*/
|
*/
|
||||||
|
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||||
public final class FrequencyTrie<V> {
|
public final class FrequencyTrie<V> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -93,6 +97,16 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName());
|
private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName());
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Domain separator used by the trie fingerprint canonical input.
|
||||||
|
*/
|
||||||
|
private static final String FINGERPRINT_DOMAIN = "RADIXOR-FREQUENCY-TRIE-FINGERPRINT";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Version of the canonical fingerprint input format.
|
||||||
|
*/
|
||||||
|
private static final int FINGERPRINT_FORMAT_VERSION = 1;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Root node of the compiled read-only trie.
|
* Root node of the compiled read-only trie.
|
||||||
*/
|
*/
|
||||||
@@ -103,6 +117,12 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
private final TrieMetadata metadata;
|
private final TrieMetadata metadata;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Canonical SHA-256 fingerprint bytes. The internal array is never exposed
|
||||||
|
* directly to callers.
|
||||||
|
*/
|
||||||
|
private final byte[] fingerprintBytes;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cached traversal direction used for key lookup.
|
* Cached traversal direction used for key lookup.
|
||||||
*/
|
*/
|
||||||
@@ -233,6 +253,7 @@ public final class FrequencyTrie<V> {
|
|||||||
final TrieMetadata metadata) {
|
final TrieMetadata metadata) {
|
||||||
this.root = Objects.requireNonNull(root, "root");
|
this.root = Objects.requireNonNull(root, "root");
|
||||||
this.metadata = Objects.requireNonNull(metadata, "metadata");
|
this.metadata = Objects.requireNonNull(metadata, "metadata");
|
||||||
|
this.fingerprintBytes = computeFingerprintBytes(root, metadata);
|
||||||
this.lookupTraversalDirection = metadata.traversalDirection();
|
this.lookupTraversalDirection = metadata.traversalDirection();
|
||||||
this.lowercasesLookupKeys = metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
this.lowercasesLookupKeys = metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||||
this.removeDiacritics = metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE;
|
this.removeDiacritics = metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE;
|
||||||
@@ -504,6 +525,60 @@ public final class FrequencyTrie<V> {
|
|||||||
return this.metadata;
|
return this.metadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the deterministic SHA-256 fingerprint of this trie.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* The fingerprint is a canonical model identity, not a Java object identity. It
|
||||||
|
* includes a fingerprint-domain marker, the fingerprint input format version,
|
||||||
|
* persisted metadata, and the complete compiled-node structure reachable from
|
||||||
|
* the root, including edges, child references, local values, and local counts.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* The returned value is stable across JVM runs for equivalent trie content and
|
||||||
|
* metadata. It does not include object identity, memory layout, runtime cache
|
||||||
|
* state, absolute file paths, timestamps, or other process-local state.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @return 64-character lowercase hexadecimal SHA-256 fingerprint
|
||||||
|
*/
|
||||||
|
public String getFingerprint() {
|
||||||
|
return toLowerHex(this.fingerprintBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a defensive copy of the raw SHA-256 fingerprint bytes.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* The returned array has length {@code 32}. Mutating it does not affect this
|
||||||
|
* trie.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @return defensive copy of the 32-byte SHA-256 fingerprint
|
||||||
|
*/
|
||||||
|
public byte[] copyFingerprintBytes() {
|
||||||
|
return Arrays.copyOf(this.fingerprintBytes, this.fingerprintBytes.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <V> byte[] computeFingerprintBytes(final CompiledNode<V> root, final TrieMetadata metadata) {
|
||||||
|
final MessageDigest messageDigest = newSha256Digest();
|
||||||
|
updateUtf8(messageDigest, FINGERPRINT_DOMAIN);
|
||||||
|
updateInt(messageDigest, FINGERPRINT_FORMAT_VERSION);
|
||||||
|
updateUtf8(messageDigest, metadata.toTextBlock());
|
||||||
|
|
||||||
|
final Map<CompiledNode<V>, Integer> nodeIds = new IdentityHashMap<>();
|
||||||
|
final List<CompiledNode<V>> orderedNodes = new ArrayList<>();
|
||||||
|
assignNodeIds(root, nodeIds, orderedNodes);
|
||||||
|
|
||||||
|
updateInt(messageDigest, nodeIds.get(root));
|
||||||
|
updateInt(messageDigest, orderedNodes.size());
|
||||||
|
for (CompiledNode<V> node : orderedNodes) {
|
||||||
|
updateNodeFingerprint(messageDigest, node, nodeIds);
|
||||||
|
}
|
||||||
|
return messageDigest.digest();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the root node mainly for diagnostics and tests within the package.
|
* Returns the root node mainly for diagnostics and tests within the package.
|
||||||
*
|
*
|
||||||
@@ -687,6 +762,63 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static MessageDigest newSha256Digest() {
|
||||||
|
try {
|
||||||
|
return MessageDigest.getInstance("SHA-256");
|
||||||
|
} catch (NoSuchAlgorithmException exception) {
|
||||||
|
throw new IllegalStateException("SHA-256 digest is not available.", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <V> void updateNodeFingerprint(final MessageDigest messageDigest, final CompiledNode<V> node,
|
||||||
|
final Map<CompiledNode<V>, Integer> nodeIds) {
|
||||||
|
final char[] edgeLabels = node.edgeLabels();
|
||||||
|
final CompiledNode<V>[] children = node.children();
|
||||||
|
final V[] values = node.orderedValues();
|
||||||
|
final int[] counts = node.orderedCounts();
|
||||||
|
|
||||||
|
updateInt(messageDigest, edgeLabels.length);
|
||||||
|
for (char edgeLabel : edgeLabels) {
|
||||||
|
updateInt(messageDigest, edgeLabel);
|
||||||
|
}
|
||||||
|
for (CompiledNode<V> child : children) {
|
||||||
|
final Integer childNodeId = nodeIds.get(child);
|
||||||
|
if (childNodeId == null) {
|
||||||
|
throw new IllegalStateException("Missing child node identifier during trie fingerprinting.");
|
||||||
|
}
|
||||||
|
updateInt(messageDigest, childNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
updateInt(messageDigest, values.length);
|
||||||
|
for (V value : values) {
|
||||||
|
updateUtf8(messageDigest, String.valueOf(value));
|
||||||
|
}
|
||||||
|
for (int count : counts) {
|
||||||
|
updateInt(messageDigest, count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void updateUtf8(final MessageDigest messageDigest, final String value) {
|
||||||
|
final byte[] encoded = value.getBytes(StandardCharsets.UTF_8);
|
||||||
|
updateInt(messageDigest, encoded.length);
|
||||||
|
messageDigest.update(encoded);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void updateInt(final MessageDigest messageDigest, final int value) {
|
||||||
|
messageDigest.update((byte) (value >>> 24));
|
||||||
|
messageDigest.update((byte) (value >>> 16));
|
||||||
|
messageDigest.update((byte) (value >>> 8));
|
||||||
|
messageDigest.update((byte) value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String toLowerHex(final byte[] digest) {
|
||||||
|
final StringBuilder builder = new StringBuilder(digest.length * 2);
|
||||||
|
for (byte item : digest) {
|
||||||
|
builder.append(Character.forDigit((item >>> 4) & 0x0F, 16)).append(Character.forDigit(item & 0x0F, 16));
|
||||||
|
}
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Internal helper that materializes serialized trie data.
|
* Internal helper that materializes serialized trie data.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -158,7 +158,7 @@ public final class CompiledNode<V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
final CompiledNode<V>[] dense = (CompiledNode<V>[]) new CompiledNode[span + 1];
|
final CompiledNode<V>[] dense = new CompiledNode[span + 1];
|
||||||
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
|
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||||
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
|
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
|
||||||
}
|
}
|
||||||
@@ -347,7 +347,7 @@ public final class CompiledNode<V> {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Finds a child for the supplied edge character.
|
* Finds a child for the supplied edge character.
|
||||||
* <p>
|
*
|
||||||
* Lookup order is:
|
* Lookup order is:
|
||||||
* <ol>
|
* <ol>
|
||||||
* <li>dense array index (if the label interval is compact enough),</li>
|
* <li>dense array index (if the label interval is compact enough),</li>
|
||||||
@@ -355,7 +355,6 @@ public final class CompiledNode<V> {
|
|||||||
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
|
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
|
||||||
* <li>binary search over sorted labels.</li>
|
* <li>binary search over sorted labels.</li>
|
||||||
* </ol>
|
* </ol>
|
||||||
* </p>
|
|
||||||
*
|
*
|
||||||
* @param edge edge character
|
* @param edge edge character
|
||||||
* @return child node, or {@code null} if absent
|
* @return child node, or {@code null} if absent
|
||||||
|
|||||||
@@ -873,6 +873,7 @@ class FrequencyTrieTest {
|
|||||||
.readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
|
.readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
|
||||||
|
|
||||||
assertAll(() -> assertEquals(original.size(), restored.size()),
|
assertAll(() -> assertEquals(original.size(), restored.size()),
|
||||||
|
() -> assertEquals(original.getFingerprint(), restored.getFingerprint()),
|
||||||
() -> assertEquals(original.get(""), restored.get("")),
|
() -> assertEquals(original.get(""), restored.get("")),
|
||||||
() -> assertArrayEquals(original.getAll(""), restored.getAll("")),
|
() -> assertArrayEquals(original.getAll(""), restored.getAll("")),
|
||||||
() -> assertEquals(original.get("run"), restored.get("run")),
|
() -> assertEquals(original.get("run"), restored.get("run")),
|
||||||
@@ -892,6 +893,82 @@ class FrequencyTrieTest {
|
|||||||
() -> assertEquals(List.of(), restored.getEntries("missing")));
|
() -> assertEquals(List.of(), restored.getEntries("missing")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies fingerprint stability and sensitivity to metadata and trie content.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Fingerprint reflects metadata and compiled trie content")
|
||||||
|
void fingerprintReflectsMetadataAndCompiledTrieContent() {
|
||||||
|
final FrequencyTrie.Builder<String> baseBuilderA = rankedBuilder();
|
||||||
|
baseBuilderA.put("run", "verb", 3);
|
||||||
|
baseBuilderA.put("run", "noun", 1);
|
||||||
|
baseBuilderA.put("runner", "noun", 2);
|
||||||
|
final FrequencyTrie<String> trieA = baseBuilderA.build();
|
||||||
|
|
||||||
|
final FrequencyTrie.Builder<String> baseBuilderB = rankedBuilder();
|
||||||
|
baseBuilderB.put("run", "verb", 3);
|
||||||
|
baseBuilderB.put("run", "noun", 1);
|
||||||
|
baseBuilderB.put("runner", "noun", 2);
|
||||||
|
final FrequencyTrie<String> trieB = baseBuilderB.build();
|
||||||
|
|
||||||
|
final FrequencyTrie.Builder<String> reorderedBuilder = rankedBuilder();
|
||||||
|
reorderedBuilder.put("runner", "noun", 2);
|
||||||
|
reorderedBuilder.put("run", "noun", 1);
|
||||||
|
reorderedBuilder.put("run", "verb", 3);
|
||||||
|
final FrequencyTrie<String> reorderedTrie = reorderedBuilder.build();
|
||||||
|
|
||||||
|
final FrequencyTrie.Builder<String> differentContentBuilder = rankedBuilder();
|
||||||
|
differentContentBuilder.put("run", "verb", 3);
|
||||||
|
differentContentBuilder.put("run", "noun", 2);
|
||||||
|
differentContentBuilder.put("runner", "noun", 2);
|
||||||
|
final FrequencyTrie<String> differentContentTrie = differentContentBuilder.build();
|
||||||
|
|
||||||
|
final FrequencyTrie.Builder<String> differentMetadataBuilder = new FrequencyTrie.Builder<>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||||
|
WordTraversalDirection.FORWARD, CaseProcessingMode.AS_IS);
|
||||||
|
differentMetadataBuilder.put("run", "verb", 3);
|
||||||
|
differentMetadataBuilder.put("run", "noun", 1);
|
||||||
|
differentMetadataBuilder.put("runner", "noun", 2);
|
||||||
|
final FrequencyTrie<String> differentMetadataTrie = differentMetadataBuilder.build();
|
||||||
|
|
||||||
|
final String fingerprintA = trieA.getFingerprint();
|
||||||
|
final String fingerprintB = trieB.getFingerprint();
|
||||||
|
final String reorderedFingerprint = reorderedTrie.getFingerprint();
|
||||||
|
final String differentContentFingerprint = differentContentTrie.getFingerprint();
|
||||||
|
final String differentMetadataFingerprint = differentMetadataTrie.getFingerprint();
|
||||||
|
final byte[] fingerprintBytes = trieA.copyFingerprintBytes();
|
||||||
|
final byte[] secondFingerprintBytes = trieA.copyFingerprintBytes();
|
||||||
|
fingerprintBytes[0] = (byte) (fingerprintBytes[0] ^ 0x7F);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(fingerprintA, fingerprintB),
|
||||||
|
() -> assertEquals(fingerprintA, reorderedFingerprint),
|
||||||
|
() -> assertEquals(fingerprintA, trieA.getFingerprint()),
|
||||||
|
() -> assertFalse(fingerprintA.isBlank()),
|
||||||
|
() -> assertLowercaseSha256Hex(fingerprintA),
|
||||||
|
() -> assertEquals(fingerprintA, toLowerHex(secondFingerprintBytes)),
|
||||||
|
() -> assertArrayEquals(secondFingerprintBytes, trieA.copyFingerprintBytes()),
|
||||||
|
() -> assertFalse(fingerprintA.equals(differentContentFingerprint)),
|
||||||
|
() -> assertFalse(fingerprintA.equals(differentMetadataFingerprint)));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertLowercaseSha256Hex(final String fingerprint) {
|
||||||
|
assertEquals(64, fingerprint.length());
|
||||||
|
for (int index = 0; index < fingerprint.length(); index++) {
|
||||||
|
final char character = fingerprint.charAt(index);
|
||||||
|
final boolean digit = character >= '0' && character <= '9';
|
||||||
|
final boolean lowercaseHex = character >= 'a' && character <= 'f';
|
||||||
|
assertTrue(digit || lowercaseHex, "Invalid fingerprint character at index " + index + '.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String toLowerHex(final byte[] bytes) {
|
||||||
|
final StringBuilder builder = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte item : bytes) {
|
||||||
|
builder.append(Character.forDigit((item >>> 4) & 0x0F, 16));
|
||||||
|
builder.append(Character.forDigit(item & 0x0F, 16));
|
||||||
|
}
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
/**
|
/**
|
||||||
* Verifies that persistence methods reject {@code null} arguments.
|
* Verifies that persistence methods reject {@code null} arguments.
|
||||||
*
|
*
|
||||||
|
|||||||
Reference in New Issue
Block a user