From 464b5804366e8d7034a819f891f5cd730ad8c005 Mon Sep 17 00:00:00 2001 From: Leo Galambos Date: Sun, 24 May 2026 19:54:29 +0200 Subject: [PATCH] feat: Add FrequencyTrie model fingerprints for EGOTHOR v4 analyzer identity The fingerprint covers trie metadata and the compiled node graph, exposes a lowercase hex representation plus defensive raw bytes, and is stable across equivalent trie builds and persistence round-trips. --- .project | 27 ++-- .../org/egothor/stemmer/FrequencyTrie.java | 133 ++++++++++++++++++ .../egothor/stemmer/trie/CompiledNode.java | 5 +- .../egothor/stemmer/FrequencyTrieTest.java | 77 ++++++++++ 4 files changed, 225 insertions(+), 17 deletions(-) diff --git a/.project b/.project index a6a5df7..5da9344 100644 --- a/.project +++ b/.project @@ -2,22 +2,21 @@ Radixor - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - + org.eclipse.jdt.core.javanature org.eclipse.buildship.core.gradleprojectnature + + + org.eclipse.jdt.core.javabuilder + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + diff --git a/src/main/java/org/egothor/stemmer/FrequencyTrie.java b/src/main/java/org/egothor/stemmer/FrequencyTrie.java index 3beac75..55fc28d 100644 --- a/src/main/java/org/egothor/stemmer/FrequencyTrie.java +++ b/src/main/java/org/egothor/stemmer/FrequencyTrie.java @@ -35,6 +35,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -93,6 +96,16 @@ public final class FrequencyTrie { */ private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName()); + /** + * Domain separator used by the trie fingerprint canonical input. + */ + private static final String FINGERPRINT_DOMAIN = "RADIXOR-FREQUENCY-TRIE-FINGERPRINT"; + + /** + * Version of the canonical fingerprint input format. + */ + private static final int FINGERPRINT_FORMAT_VERSION = 1; + /** * Root node of the compiled read-only trie. */ @@ -103,6 +116,12 @@ public final class FrequencyTrie { */ private final TrieMetadata metadata; + /** + * Canonical SHA-256 fingerprint bytes. The internal array is never exposed + * directly to callers. + */ + private final byte[] fingerprintBytes; + /** * Cached traversal direction used for key lookup. */ @@ -233,6 +252,7 @@ public final class FrequencyTrie { final TrieMetadata metadata) { this.root = Objects.requireNonNull(root, "root"); this.metadata = Objects.requireNonNull(metadata, "metadata"); + this.fingerprintBytes = computeFingerprintBytes(root, metadata); this.lookupTraversalDirection = metadata.traversalDirection(); this.lowercasesLookupKeys = metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT; this.removeDiacritics = metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE; @@ -504,6 +524,61 @@ public final class FrequencyTrie { return this.metadata; } + /** + * Returns the deterministic SHA-256 fingerprint of this trie. + * + *

+ * The fingerprint is a canonical model identity, not a Java object identity. + * It includes a fingerprint-domain marker, the fingerprint input format + * version, persisted metadata, and the complete compiled-node structure + * reachable from the root, including edges, child references, local values, + * and local counts. + *

+ * + *

+ * The returned value is stable across JVM runs for equivalent trie content and + * metadata. It does not include object identity, memory layout, runtime cache + * state, absolute file paths, timestamps, or other process-local state. + *

+ * + * @return 64-character lowercase hexadecimal SHA-256 fingerprint + */ + public String getFingerprint() { + return toLowerHex(this.fingerprintBytes); + } + + /** + * Returns a defensive copy of the raw SHA-256 fingerprint bytes. + * + *

+ * The returned array has length {@code 32}. Mutating it does not affect this + * trie. + *

+ * + * @return defensive copy of the 32-byte SHA-256 fingerprint + */ + public byte[] copyFingerprintBytes() { + return Arrays.copyOf(this.fingerprintBytes, this.fingerprintBytes.length); + } + + private static byte[] computeFingerprintBytes(final CompiledNode root, final TrieMetadata metadata) { + final MessageDigest messageDigest = newSha256Digest(); + updateUtf8(messageDigest, FINGERPRINT_DOMAIN); + updateInt(messageDigest, FINGERPRINT_FORMAT_VERSION); + updateUtf8(messageDigest, metadata.toTextBlock()); + + final Map, Integer> nodeIds = new IdentityHashMap<>(); + final List> orderedNodes = new ArrayList<>(); + assignNodeIds(root, nodeIds, orderedNodes); + + updateInt(messageDigest, nodeIds.get(root)); + updateInt(messageDigest, orderedNodes.size()); + for (CompiledNode node : orderedNodes) { + updateNodeFingerprint(messageDigest, node, nodeIds); + } + return messageDigest.digest(); + } + /** * Returns the root node mainly for diagnostics and tests within the package. * @@ -687,6 +762,64 @@ public final class FrequencyTrie { } } + private static MessageDigest newSha256Digest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 digest is not available.", exception); + } + } + + private static void updateNodeFingerprint(final MessageDigest messageDigest, final CompiledNode node, + final Map, Integer> nodeIds) { + final char[] edgeLabels = node.edgeLabels(); + final CompiledNode[] children = node.children(); + final V[] values = node.orderedValues(); + final int[] counts = node.orderedCounts(); + + updateInt(messageDigest, edgeLabels.length); + for (char edgeLabel : edgeLabels) { + updateInt(messageDigest, edgeLabel); + } + for (CompiledNode child : children) { + final Integer childNodeId = nodeIds.get(child); + if (childNodeId == null) { + throw new IllegalStateException("Missing child node identifier during trie fingerprinting."); + } + updateInt(messageDigest, childNodeId.intValue()); + } + + updateInt(messageDigest, values.length); + for (V value : values) { + updateUtf8(messageDigest, String.valueOf(value)); + } + for (int count : counts) { + updateInt(messageDigest, count); + } + } + + private static void updateUtf8(final MessageDigest messageDigest, final String value) { + final byte[] encoded = value.getBytes(StandardCharsets.UTF_8); + updateInt(messageDigest, encoded.length); + messageDigest.update(encoded); + } + + private static void updateInt(final MessageDigest messageDigest, final int value) { + messageDigest.update((byte) (value >>> 24)); + messageDigest.update((byte) (value >>> 16)); + messageDigest.update((byte) (value >>> 8)); + messageDigest.update((byte) value); + } + + private static String toLowerHex(final byte[] digest) { + final StringBuilder builder = new StringBuilder(digest.length * 2); + for (byte item : digest) { + builder.append(Character.forDigit((item >>> 4) & 0x0F, 16)); + builder.append(Character.forDigit(item & 0x0F, 16)); + } + return builder.toString(); + } + /** * Internal helper that materializes serialized trie data. * diff --git a/src/main/java/org/egothor/stemmer/trie/CompiledNode.java b/src/main/java/org/egothor/stemmer/trie/CompiledNode.java index 4aad015..8d527f7 100644 --- a/src/main/java/org/egothor/stemmer/trie/CompiledNode.java +++ b/src/main/java/org/egothor/stemmer/trie/CompiledNode.java @@ -158,7 +158,7 @@ public final class CompiledNode { } @SuppressWarnings("unchecked") - final CompiledNode[] dense = (CompiledNode[]) new CompiledNode[span + 1]; + final CompiledNode[] dense = new CompiledNode[span + 1]; for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) { dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex]; } @@ -347,7 +347,7 @@ public final class CompiledNode { /** * Finds a child for the supplied edge character. - *

+ * * Lookup order is: *

    *
  1. dense array index (if the label interval is compact enough),
    2. @@ -355,7 +355,6 @@ public final class CompiledNode { * {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges, *
  2. binary search over sorted labels.
    3. *
- *

* * @param edge edge character * @return child node, or {@code null} if absent diff --git a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java index 91059c5..67220f2 100644 --- a/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java +++ b/src/test/java/org/egothor/stemmer/FrequencyTrieTest.java @@ -873,6 +873,7 @@ class FrequencyTrieTest { .readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC); assertAll(() -> assertEquals(original.size(), restored.size()), + () -> assertEquals(original.getFingerprint(), restored.getFingerprint()), () -> assertEquals(original.get(""), restored.get("")), () -> assertArrayEquals(original.getAll(""), restored.getAll("")), () -> assertEquals(original.get("run"), restored.get("run")), @@ -892,6 +893,82 @@ class FrequencyTrieTest { () -> assertEquals(List.of(), restored.getEntries("missing"))); } + /** + * Verifies fingerprint stability and sensitivity to metadata and trie content. + */ + @Test + @DisplayName("Fingerprint reflects metadata and compiled trie content") + void fingerprintReflectsMetadataAndCompiledTrieContent() { + final FrequencyTrie.Builder baseBuilderA = rankedBuilder(); + baseBuilderA.put("run", "verb", 3); + baseBuilderA.put("run", "noun", 1); + baseBuilderA.put("runner", "noun", 2); + final FrequencyTrie trieA = baseBuilderA.build(); + + final FrequencyTrie.Builder baseBuilderB = rankedBuilder(); + baseBuilderB.put("run", "verb", 3); + baseBuilderB.put("run", "noun", 1); + baseBuilderB.put("runner", "noun", 2); + final FrequencyTrie trieB = baseBuilderB.build(); + + final FrequencyTrie.Builder reorderedBuilder = rankedBuilder(); + reorderedBuilder.put("runner", "noun", 2); + reorderedBuilder.put("run", "noun", 1); + reorderedBuilder.put("run", "verb", 3); + final FrequencyTrie reorderedTrie = reorderedBuilder.build(); + + final FrequencyTrie.Builder differentContentBuilder = rankedBuilder(); + differentContentBuilder.put("run", "verb", 3); + differentContentBuilder.put("run", "noun", 2); + differentContentBuilder.put("runner", "noun", 2); + final FrequencyTrie differentContentTrie = differentContentBuilder.build(); + + final FrequencyTrie.Builder differentMetadataBuilder = new FrequencyTrie.Builder<>(String[]::new, + ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS), + WordTraversalDirection.FORWARD, CaseProcessingMode.AS_IS); + differentMetadataBuilder.put("run", "verb", 3); + differentMetadataBuilder.put("run", "noun", 1); + differentMetadataBuilder.put("runner", "noun", 2); + final FrequencyTrie differentMetadataTrie = differentMetadataBuilder.build(); + + final String fingerprintA = trieA.getFingerprint(); + final String fingerprintB = trieB.getFingerprint(); + final String reorderedFingerprint = reorderedTrie.getFingerprint(); + final String differentContentFingerprint = differentContentTrie.getFingerprint(); + final String differentMetadataFingerprint = differentMetadataTrie.getFingerprint(); + final byte[] fingerprintBytes = trieA.copyFingerprintBytes(); + final byte[] secondFingerprintBytes = trieA.copyFingerprintBytes(); + fingerprintBytes[0] = (byte) (fingerprintBytes[0] ^ 0x7F); + + assertAll(() -> assertEquals(fingerprintA, fingerprintB), + () -> assertEquals(fingerprintA, reorderedFingerprint), + () -> assertEquals(fingerprintA, trieA.getFingerprint()), + () -> assertFalse(fingerprintA.isBlank()), + () -> assertLowercaseSha256Hex(fingerprintA), + () -> assertEquals(fingerprintA, toLowerHex(secondFingerprintBytes)), + () -> assertArrayEquals(secondFingerprintBytes, trieA.copyFingerprintBytes()), + () -> assertFalse(fingerprintA.equals(differentContentFingerprint)), + () -> assertFalse(fingerprintA.equals(differentMetadataFingerprint))); + } + + private static void assertLowercaseSha256Hex(final String fingerprint) { + assertEquals(64, fingerprint.length()); + for (int index = 0; index < fingerprint.length(); index++) { + final char character = fingerprint.charAt(index); + final boolean digit = character >= '0' && character <= '9'; + final boolean lowercaseHex = character >= 'a' && character <= 'f'; + assertTrue(digit || lowercaseHex, "Invalid fingerprint character at index " + index + '.'); + } + } + + private static String toLowerHex(final byte[] bytes) { + final StringBuilder builder = new StringBuilder(bytes.length * 2); + for (byte item : bytes) { + builder.append(Character.forDigit((item >>> 4) & 0x0F, 16)); + builder.append(Character.forDigit(item & 0x0F, 16)); + } + return builder.toString(); + } /** * Verifies that persistence methods reject {@code null} arguments. *