Compare commits

...

10 Commits

Author SHA1 Message Date
df4552b113 fix(javadoc): Remove collision block 2026-05-24 20:14:28 +02:00
9a84add263 fix(pmd): PMD errors fixed 2026-05-24 20:09:11 +02:00
1a02c41348 feat: Add FrequencyTrie model fingerprints for EGOTHOR v4 analyzer identity
The fingerprint covers trie metadata and the compiled node graph, exposes
a lowercase hex representation plus defensive raw bytes, and is stable
across equivalent trie builds and persistence round-trips.
2026-05-24 20:05:03 +02:00
464b580436 feat: Add FrequencyTrie model fingerprints for EGOTHOR v4 analyzer identity
The fingerprint covers trie metadata and the compiled node graph, exposes
a lowercase hex representation plus defensive raw bytes, and is stable
across equivalent trie builds and persistence round-trips.
2026-05-24 19:54:29 +02:00
b945902f05 feat: Add JPMS module descriptor for org.egothor.radixor 2026-05-17 17:19:09 +02:00
902ad117e8 fix(docs): new header/banner in README 2026-05-17 16:14:23 +02:00
14a1e2fc53 fix(test): JUnit tagging and coverage 2026-05-17 16:02:35 +02:00
87ff85fd6d feat: EGOTHOR v4 hot-path additions 2026-05-17 15:00:45 +02:00
7bd0fc66ba fix: workflow indent typo 2026-05-16 03:27:00 +02:00
dadab5514e feat: implement dense-child optimized trie lookup and enterprise test/CI profile hardening 2026-05-16 03:24:07 +02:00
54 changed files with 4203 additions and 328 deletions

View File

@@ -51,7 +51,7 @@ jobs:
test -f gradle/verification-metadata.xml test -f gradle/verification-metadata.xml
- name: Execute build, tests, PMD, coverage, Javadoc, distribution packaging, and SBOM generation - name: Execute build, tests, PMD, coverage, Javadoc, distribution packaging, and SBOM generation
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport distZip cyclonedxBom run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom
- name: Upload SBOM - name: Upload SBOM
if: always() if: always()
@@ -70,8 +70,8 @@ jobs:
with: with:
name: test-reports name: test-reports
path: | path: |
build/reports/tests/test build/reports/tests
build/test-results/test build/test-results
if-no-files-found: warn if-no-files-found: warn
retention-days: 14 retention-days: 14
@@ -90,8 +90,8 @@ jobs:
with: with:
name: coverage-reports name: coverage-reports
path: | path: |
build/reports/jacoco/test/html build/reports/jacoco/jacocoCiReleaseReport/html
build/reports/jacoco/test/jacocoTestReport.xml build/reports/jacoco/jacocoCiReleaseReport/jacocoCiReleaseReport.xml
if-no-files-found: warn if-no-files-found: warn
retention-days: 14 retention-days: 14
@@ -160,7 +160,7 @@ jobs:
env: env:
SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport cyclonedxBom centralBundle run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom centralBundle
- name: Generate release changelog - name: Generate release changelog
shell: bash shell: bash

View File

@@ -70,7 +70,7 @@ jobs:
test -f gradle/verification-metadata.xml test -f gradle/verification-metadata.xml
- name: Build reports for publication - name: Build reports for publication
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport pitest jmh cyclonedxBom run: ./gradlew --no-daemon clean ciRelease pmdMain javadoc jacocoCiReleaseReport pitest jmh cyclonedxBom
- name: Prepare gh-pages worktree - name: Prepare gh-pages worktree
shell: bash shell: bash
@@ -93,6 +93,9 @@ jobs:
run: | run: |
set -euo pipefail set -euo pipefail
TEST_REPORT_DIR="build/reports/tests/ciRelease"
JACOCO_REPORT_DIR="build/reports/jacoco/jacocoCiReleaseReport"
SITE_DIR=".gh-pages" SITE_DIR=".gh-pages"
RUN_DIR="${SITE_DIR}/builds/${GITHUB_RUN_NUMBER}" RUN_DIR="${SITE_DIR}/builds/${GITHUB_RUN_NUMBER}"
RUN_METRICS_DIR="${RUN_DIR}/metrics" RUN_METRICS_DIR="${RUN_DIR}/metrics"
@@ -106,14 +109,14 @@ jobs:
cp -R build/docs/javadoc "${RUN_DIR}/javadoc" cp -R build/docs/javadoc "${RUN_DIR}/javadoc"
cp -R build/docs/javadoc "${LATEST_DIR}/javadoc" cp -R build/docs/javadoc "${LATEST_DIR}/javadoc"
cp -R build/reports/tests/test "${RUN_DIR}/test" cp -R "${TEST_REPORT_DIR}" "${RUN_DIR}/test"
cp -R build/reports/tests/test "${LATEST_DIR}/test" cp -R "${TEST_REPORT_DIR}" "${LATEST_DIR}/test"
cp -R build/reports/pmd "${RUN_DIR}/pmd" cp -R build/reports/pmd "${RUN_DIR}/pmd"
cp -R build/reports/pmd "${LATEST_DIR}/pmd" cp -R build/reports/pmd "${LATEST_DIR}/pmd"
cp -R build/reports/jacoco/test/html "${RUN_DIR}/coverage" cp -R "${JACOCO_REPORT_DIR}/html" "${RUN_DIR}/coverage"
cp -R build/reports/jacoco/test/html "${LATEST_DIR}/coverage" cp -R "${JACOCO_REPORT_DIR}/html" "${LATEST_DIR}/coverage"
cp -R build/reports/pitest "${RUN_DIR}/pitest" cp -R build/reports/pitest "${RUN_DIR}/pitest"
cp -R build/reports/pitest "${LATEST_DIR}/pitest" cp -R build/reports/pitest "${LATEST_DIR}/pitest"
@@ -178,7 +181,7 @@ jobs:
python3 \ python3 \
./tools/generate-pages-badges.py \ ./tools/generate-pages-badges.py \
--jacoco-xml build/reports/jacoco/test/jacocoTestReport.xml \ --jacoco-xml "${JACOCO_REPORT_DIR}/jacocoCiReleaseReport.xml" \
--pit-xml build/reports/pitest/mutations.xml \ --pit-xml build/reports/pitest/mutations.xml \
--jmh-csv build/reports/jmh/jmh-results.csv \ --jmh-csv build/reports/jmh/jmh-results.csv \
--run-metrics-dir "${RUN_METRICS_DIR}" \ --run-metrics-dir "${RUN_METRICS_DIR}" \
@@ -228,7 +231,7 @@ jobs:
<p class="meta">Build ${GITHUB_RUN_NUMBER} from commit ${GITHUB_SHA}</p> <p class="meta">Build ${GITHUB_RUN_NUMBER} from commit ${GITHUB_SHA}</p>
<ul> <ul>
<li><a href="./javadoc/">Javadoc</a></li> <li><a href="./javadoc/">Javadoc</a></li>
<li><a href="./test/">Test Report</a></li> <li><a href="./test/">Release Verification Test Report (ciRelease)</a></li>
<li><a href="./pmd/main.html">PMD Report</a></li> <li><a href="./pmd/main.html">PMD Report</a></li>
<li><a href="./coverage/">Coverage Report</a></li> <li><a href="./coverage/">Coverage Report</a></li>
${DEPENDENCY_CHECK_LINK:-<li>Dependency Vulnerability Report: not available</li>} ${DEPENDENCY_CHECK_LINK:-<li>Dependency Vulnerability Report: not available</li>}
@@ -260,7 +263,7 @@ jobs:
- [Latest build summary](https://leogalambos.github.io/Radixor/builds/latest/) - [Latest build summary](https://leogalambos.github.io/Radixor/builds/latest/)
- [Javadoc](https://leogalambos.github.io/Radixor/builds/latest/javadoc/) - [Javadoc](https://leogalambos.github.io/Radixor/builds/latest/javadoc/)
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/) - [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html) - [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/) - [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/) - [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)

View File

@@ -162,7 +162,7 @@
<rule ref="category/java/design.xml/CollapsibleIfStatements"/> <rule ref="category/java/design.xml/CollapsibleIfStatements"/>
<rule ref="category/java/design.xml/CouplingBetweenObjects"> <rule ref="category/java/design.xml/CouplingBetweenObjects">
<properties> <properties>
<property name="threshold" value="60" /> <property name="threshold" value="70" />
</properties> </properties>
</rule> </rule>
<rule ref="category/java/design.xml/CyclomaticComplexity"> <rule ref="category/java/design.xml/CyclomaticComplexity">

View File

@@ -1,15 +1,13 @@
<img src="Radixor.png" width="30%" align="right" alt="Radixor logo" /> <img src="docs/assets/images/banner.jpg" width="100%" alt="Radixor banner" />
# Radixor
[![Quality gates](https://github.com/leogalambos/Radixor/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/leogalambos/Radixor/actions/workflows/build.yml)
[![Coverage](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
[![Published reports](https://img.shields.io/badge/reports-GitHub%20Pages-blue)](https://leogalambos.github.io/Radixor/builds/latest/)
[![Mutation score](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
[![English benchmark](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/jmh-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt)
[![Maven Central](https://img.shields.io/maven-central/v/org.egothor/radixor)](https://central.sonatype.com/artifact/org.egothor/radixor)
[![License](https://img.shields.io/github/license/leogalambos/Radixor)](LICENSE) [![License](https://img.shields.io/github/license/leogalambos/Radixor)](LICENSE)
[![Java](https://img.shields.io/badge/Java-21%2B-brightgreen)](#) [![Java](https://img.shields.io/badge/Java-21%2B-brightgreen)](#)
[![Maven Central](https://img.shields.io/maven-central/v/org.egothor/radixor)](https://central.sonatype.com/artifact/org.egothor/radixor)
[![Published reports](https://img.shields.io/badge/reports-GitHub%20Pages-blue)](https://leogalambos.github.io/Radixor/builds/latest/)
[![Quality gates](https://github.com/leogalambos/Radixor/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/leogalambos/Radixor/actions/workflows/build.yml)
[![Coverage](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/coverage-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
[![Mutation score](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/pitest-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
[![English benchmark](https://img.shields.io/endpoint?url=https://leogalambos.github.io/Radixor/builds/latest/metrics/jmh-badge.json)](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt)
*Fast, deterministic, multi-language stemming for Java, built around compact patch-command tries and measured at roughly 4× to 6× the throughput of the Snowball Porter stemmer family on the current English benchmark workload.* *Fast, deterministic, multi-language stemming for Java, built around compact patch-command tries and measured at roughly 4× to 6× the throughput of the Snowball Porter stemmer family on the current English benchmark workload.*
@@ -167,6 +165,9 @@ The repository keeps the front page concise and places detailed documentation un
- [Architecture](docs/architecture.md) - [Architecture](docs/architecture.md)
Structural model, data flow, and runtime lookup behavior. Structural model, data flow, and runtime lookup behavior.
- [Lookup Edge Optimization](docs/lookup-edge-optimization.md)
Speed/memory trade-off of dense child edge lookup in compiled tries.
- [Reduction Semantics](docs/reduction-semantics.md) - [Reduction Semantics](docs/reduction-semantics.md)
Ranked, unordered, and dominant reduction behavior. Ranked, unordered, and dominant reduction behavior.

View File

@@ -108,9 +108,19 @@ dependencyCheck {
} }
} }
tasks.withType(Test).configureEach { def cliIncludeTags = project.findProperty('includeTags')?.toString() ?: System.getProperty('includeTags')
useJUnitPlatform() def cliExcludeTags = project.findProperty('excludeTags')?.toString() ?: System.getProperty('excludeTags')
def splitTagExpression = { String tagsExpr ->
if (tagsExpr == null || tagsExpr.isBlank()) {
return []
}
return tagsExpr.split(',')
.collect { it.trim() }
.findAll { it != null && !it.isBlank() }
}
tasks.withType(Test).configureEach {
doFirst { doFirst {
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}" jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
} }
@@ -123,14 +133,127 @@ tasks.withType(Test).configureEach {
minHeapSize = '1g' minHeapSize = '1g'
maxHeapSize = '4g' maxHeapSize = '4g'
finalizedBy(tasks.named('jacocoTestReport'))
reports { reports {
junitXml.required = true junitXml.required = true
html.required = true html.required = true
} }
} }
def configureJUnitPlatformTags = { Test task, String includeTagsExpr, String excludeTagsExpr ->
task.useJUnitPlatform {
final def includes = splitTagExpression(includeTagsExpr)
final def excludes = splitTagExpression(excludeTagsExpr)
if (!includes.isEmpty()) {
includeTags(*includes.toArray(new String[0]))
}
if (!excludes.isEmpty()) {
excludeTags(*excludes.toArray(new String[0]))
}
}
}
tasks.named('test', Test) {
configureJUnitPlatformTags(it, cliIncludeTags, cliExcludeTags)
finalizedBy(tasks.named('jacocoTestReport'))
}
def configureTaggedTestProfile = { String taskName, String includeTagsExpr, String excludeTagsExpr = null,
String taskDescription = null, String testNameExcludePatterns = null ->
tasks.register(taskName, Test) {
group = 'verification'
description = taskDescription
configureJUnitPlatformTags(delegate as Test, includeTagsExpr, excludeTagsExpr)
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
dependsOn(tasks.named('compileTestJava'))
doFirst {
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
}
if (testNameExcludePatterns != null && !testNameExcludePatterns.isBlank()) {
filter {
testNameExcludePatterns.split(',').each { String pattern ->
final def trimmedPattern = pattern.trim()
if (!trimmedPattern.isEmpty()) {
excludeTestsMatching(trimmedPattern)
}
}
}
}
minHeapSize = '1g'
maxHeapSize = '4g'
reports {
junitXml.required = true
html.required = true
}
}
}
configureTaggedTestProfile(
'ciSmoke',
'unit',
'slow',
'Fast feedback profile for unit tests with slow tests explicitly excluded.',
'org.egothor.stemmer.CompileIntegrationTest*'
)
configureTaggedTestProfile(
'ciCore',
'unit,trie,frequency-trie,property',
null,
'Focused profile for core trie behavior and trie-specific property checks.'
)
configureTaggedTestProfile(
'ciIntegration',
'integration',
'slow',
'Integration pipeline profile (loader/parser/CLI/IO end-to-end flows) excluding slow integration paths.'
)
configureTaggedTestProfile(
'ciSlow',
'slow',
null,
'Targeted profile for all slow tests (large dictionaries, long-running corpus validation, and heavy integration checks).'
)
configureTaggedTestProfile(
'ciCompat',
'compat,regression',
null,
'Compatibility profile guarding persisted artifact and compatibility regressions.'
)
configureTaggedTestProfile(
'ciRelease',
null,
'slow',
'Release-profile validation of all non-slow tests.',
'org.egothor.stemmer.CompileIntegrationTest*,org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*'
)
configureTaggedTestProfile(
'ciNightly',
'fuzz',
null,
'Nightly robustness profile with fuzz testing emphasis.'
)
tasks.register('ci') {
group = 'verification'
description = 'Runs the full enterprise CI profile set in sequence.'
dependsOn(tasks.named('ciSmoke'))
dependsOn(tasks.named('ciCore'))
dependsOn(tasks.named('ciIntegration'))
dependsOn(tasks.named('ciCompat'))
}
tasks.withType(Pmd).configureEach { tasks.withType(Pmd).configureEach {
reports { reports {
xml.required = true xml.required = true
@@ -155,6 +278,36 @@ tasks.named('jacocoTestReport', JacocoReport) {
} }
} }
def registerJacocoProfileReport = { String reportTaskName, String sourceTaskName ->
tasks.register(reportTaskName, JacocoReport) {
group = 'verification'
description = "Generates Jacoco report for ${sourceTaskName} execution."
dependsOn(tasks.named(sourceTaskName))
classDirectories.setFrom(
files(sourceSets.main.output).asFileTree.matching {
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
exclude 'org/egothor/stemmer/DiacriticStripper*'
}
)
executionData.setFrom(
fileTree(layout.buildDirectory.dir('jacoco')) {
include "${sourceTaskName}.exec"
}
)
reports {
xml.required = true
csv.required = false
html.required = true
}
}
}
registerJacocoProfileReport('jacocoCiReleaseReport', 'ciRelease')
tasks.named('check') { tasks.named('check') {
dependsOn(tasks.named('jacocoTestReport')) dependsOn(tasks.named('jacocoTestReport'))
// no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze')) // no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze'))

View File

@@ -75,6 +75,14 @@ The distinction between preferred-result lookup and multi-result lookup is part
That model is part of how the public API should be understood. That model is part of how the public API should be understood.
Visitor lookup methods such as `getAllNormalized(..., EntrySink, maxResults)` are additive hot-path APIs. They expose the same local ordering and count semantics without allocating result containers, but they do not replace `get()`, `getAll()`, or `getEntries()`.
Compiled `FrequencyTrie` instances are immutable and thread-safe for concurrent reads. Visitor sinks are caller-owned and are not retained by the trie. Stored values passed to sinks are the model-owned trie values; for `FrequencyTrie<String>` patch tries, those patch strings are immutable stored strings rather than fresh per-result strings.
### Stable patch application behavior
`PatchCommandEncoder.apply(...)` remains the compatibility API for string-returning patch application. Buffer-oriented `applyTo(...)` overloads are additive APIs for caller-owned output storage. They do not retain output arrays, report insufficient capacity with `APPLY_INSUFFICIENT_CAPACITY`, and preserve the existing malformed-patch compatibility behavior where `apply(...)` preserves the source.
### Stable reduction-mode intent ### Stable reduction-mode intent
Each public `ReductionMode` constant carries a semantic contract that should remain meaningful across versions. Each public `ReductionMode` constant carries a semantic contract that should remain meaningful across versions.

View File

@@ -0,0 +1,193 @@
# Lookup Edge Optimization
Compiled trie nodes (`CompiledNode`) use three lookup strategies when resolving child edges:
1. dense array direct lookup,
2. linear scan for very small child counts,
3. binary search over sorted edge labels.
This page explains the dense path, what `maxExpandedIndex` controls, and how to tune it.
## Runtime model of one node
For a node with sorted edge labels `char[] edges`, the implementation can materialize an
index-aligned dense table when labels occupy a small compact code-point interval:
```text
span = maxEdge - minEdge
use dense table iff (span <= maxExpandedIndex) and (maxExpandedIndex > 0)
```
When dense lookup is used, lookup is constant-time indexing:
```text
denseIndex = requestedEdge - minEdge
return denseChildren[denseIndex] // or null if outside interval
```
When dense lookup is not active (interval is too wide or the configured
`maxExpandedIndex` is `0`), `CompiledNode` still chooses between two fallback
strategies:
- **linear scan** for very small child counts (`4` or fewer children),
- **binary search** for larger child counts.
This means the fallback method is selected by child count, not by “distance” alone.
`linear scan` is therefore used when there are only a few edges even if those edges are
spread across very distant code points.
### Example: few edges, wide Unicode span
```text
edges = ['a', '中', '你']
edge count = 3
minEdge = 'a' (U+0061)
maxEdge = '你' (U+4F60)
span = 20319
```
- If `maxExpandedIndex = 512`, dense indexing is not used because `span > maxExpandedIndex`.
- Because `edge count = 3` (<= 4), lookup falls back to a tiny linear scan of the
three labels.
- This is exactly the case where you get benefit from the threshold even though the interval is wide.
This is useful for non-Latin scripts as well: what matters is interval width in Unicode
code points, not script name. A compact Arabic-range block can still benefit from dense
lookups when keys stay in a tight code-point interval.
## Why this is configurable
`maxExpandedIndex` is only a performance/paging choice:
- higher value:
- more compact intervals qualify for dense tables,
- more constant-time child lookup,
- more memory for dense tables in qualifying nodes.
- lower value (or `0`):
- less dense-table allocation,
- fewer branches into constant-time path,
- lower materialization memory.
The value never changes lookup semantics. It only changes the in-memory structure shape.
## Persistence and loading model
This threshold is **not** stored in `TrieMetadata`.
- The binary format stores only trie payload and semantic metadata (`reduction`, `traversal`,
case/diacritic settings, and stream version).
- `maxExpandedIndex` is chosen when materializing nodes in memory.
- You can therefore keep one persisted artifact and load it with different in-memory
trade-offs depending on deployment constraints.
## Default
- `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX == 512`
- `CompiledNode.DEFAULT_MAX_EXPANDED_INDEX == 512`
These are practical defaults for mixed-language text and Latin-like scripts where edge labels
often cluster.
## Tune during build (writable phase)
Use the full `FrequencyTrie.Builder` constructor when you are compiling from source data.
The builder threshold is applied while freezing reduced nodes into the immutable form.
```java
import org.egothor.stemmer.CaseProcessingMode;
import org.egothor.stemmer.DiacriticProcessingMode;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.ReductionSettings;
import org.egothor.stemmer.WordTraversalDirection;
final ReductionSettings settings = ReductionSettings.withDefaults(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
final FrequencyTrie.Builder<String> fastBuilder =
new FrequencyTrie.Builder<>(String[]::new,
settings,
WordTraversalDirection.BACKWARD,
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
DiacriticProcessingMode.AS_IS,
1024); // prefer lookup speed
// ... put(...) ...
final FrequencyTrie<String> trie = fastBuilder.build();
```
Use `0` or `256` for lower memory while still building larger tries.
```java
final FrequencyTrie.Builder<String> compactBuilder =
new FrequencyTrie.Builder<>(String[]::new,
settings,
WordTraversalDirection.BACKWARD,
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
DiacriticProcessingMode.AS_IS,
256); // lower memory profile
```
## Tune when loading a binary artifact (runtime phase)
At artifact load time, you can tune the same trade-off independently of persisted metadata.
```java
import java.nio.file.Path;
import org.egothor.stemmer.StemmerPatchTrieLoader;
var defaultLookup = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"));
var fastLookup = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"), 1024);
var compactLookup = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"), 0);
```
You can also set the threshold directly with `FrequencyTrie.readFrom(...)` when reading streams:
```java
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.zip.GZIPInputStream;
import org.egothor.stemmer.FrequencyTrie;
public final class StreamLoadExample {
private StreamLoadExample() {
throw new AssertionError("No instances.");
}
public static void main(final String[] arguments) throws IOException {
try (InputStream fileInput = Files.newInputStream(Path.of("stemmers", "english.radixor.gz"));
GZIPInputStream gzip = new GZIPInputStream(fileInput);
DataInputStream dataInput = new DataInputStream(gzip)) {
final FrequencyTrie<String> compactOnLoad = FrequencyTrie.readFrom(
dataInput,
String[]::new,
input -> input.readUTF(),
256);
}
}
}
```
Note: the string codec is intentionally inline in this snippet to keep it self-contained.
## Practical guidance
- Start with default (`512`) in production and profile before changing it.
- Use `0` when memory is the priority and query throughput is not the bottleneck.
- Use values around `1024` for workloads dominated by compact alphabets and very hot lookups.
Trade-off expectation:
- increasing `maxExpandedIndex` improves lookup speed when edges tend to occupy short spans,
- decreasing it reduces per-node auxiliary memory in dense-span nodes.

View File

@@ -87,6 +87,43 @@ public final class LoadBinaryExample {
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled. The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
## Tune child lookup density when loading binaries
To optimize hot-path latency, you can tune direct child indexing by passing `maxExpandedIndex`
at load time. This does not change persisted metadata, only the materialized in-memory form.
```java
import java.io.IOException;
import java.nio.file.Path;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.StemmerPatchTrieLoader;
public final class LoadBinaryWithDenseLookupExample {
private LoadBinaryWithDenseLookupExample() {
throw new AssertionError("No instances.");
}
public static void main(final String[] arguments) throws IOException {
final FrequencyTrie<String> balanced = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"));
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"),
1024);
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"),
0);
}
}
```
Negative values still use `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX`.
[Lookup Edge Optimization](lookup-edge-optimization.md) describes the trade-off in detail and examples for build-time tuning as well.
## Build directly with a mutable builder ## Build directly with a mutable builder
A `FrequencyTrie.Builder<V>` accepts repeated `put(key, value)` calls and compiles the final read-only trie through `build()`. Compilation performs bottom-up reduction and produces the compact immutable runtime representation. A `FrequencyTrie.Builder<V>` accepts repeated `put(key, value)` calls and compiles the final read-only trie through `build()`. Compilation performs bottom-up reduction and produces the compact immutable runtime representation.

View File

@@ -33,6 +33,28 @@ import org.egothor.stemmer.ValueCount;
final List<ValueCount<String>> entries = trie.getEntries("axes"); final List<ValueCount<String>> entries = trie.getEntries("axes");
``` ```
### Visitor lookup for hot paths
For allocation-sensitive token loops, use the visitor-style lookup methods. They visit the same ordered local values and counts without allocating a result array, list, or `ValueCount` objects.
```java
trie.getAll("axes", (patch, count, rank) -> {
// rank is zero-based and follows the same ordering as getAll(String).
return true; // return false to stop after this callback
}, 8);
```
If the caller has already normalized the input exactly as required by `trie.metadata()`, the normalized methods avoid lookup normalization buffers too:
```java
final char[] token = "axes".toCharArray();
trie.getAllNormalized(token, 0, token.length, (patch, count, rank) -> {
return true;
}, 8);
```
`getAllNormalized(...)` bypasses `caseProcessingMode` and `diacriticProcessingMode`; callers are responsible for supplying canonical input. `maxResults == 0` visits nothing, negative values are rejected, and a sink returning `false` stops iteration after the current callback.
## Apply patch commands ## Apply patch commands
A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch. A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch.
@@ -45,6 +67,25 @@ final String patch = trie.get(word);
final String stem = PatchCommandEncoder.apply(word, patch); final String stem = PatchCommandEncoder.apply(word, patch);
``` ```
Hot paths can apply a patch into caller-owned character storage:
```java
final char[] output = new char[32];
final int produced = PatchCommandEncoder.applyTo(
word,
patch,
trie.traversalDirection(),
output,
0,
output.length);
if (produced != PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
final String stem = new String(output, 0, produced);
}
```
`applyTo(...)` returns the produced character count on success and `APPLY_INSUFFICIENT_CAPACITY` when the output range is too small. Capacity failure does not write partial output. The source and output ranges of the `char[]` overload must not overlap.
For multiple candidates: For multiple candidates:
```java ```java

View File

@@ -25,6 +25,7 @@ This is why Radixor can generalize beyond explicitly listed forms and why compil
The programmatic API is easier to understand when split by developer task: The programmatic API is easier to understand when split by developer task:
- [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage. - [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage.
- [Lookup Edge Optimization](lookup-edge-optimization.md) explains dense child lookup tuning and the speed/memory trade-off when materializing compiled tries.
- [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes. - [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes.
- [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) explains how to reopen compiled tries, add new lexical data, rebuild them, and store them as binary artifacts. - [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) explains how to reopen compiled tries, add new lexical data, rebuild them, and store them as binary artifacts.
@@ -40,6 +41,24 @@ The main types involved in programmatic usage are:
- `FrequencyTrieBuilders` for reconstructing a mutable builder from a compiled trie, - `FrequencyTrieBuilders` for reconstructing a mutable builder from a compiled trie,
- `ReductionMode` and `ReductionSettings` for controlling compilation semantics. - `ReductionMode` and `ReductionSettings` for controlling compilation semantics.
## Java module system (JPMS)
The core artifact is published as an explicit JPMS module:
```java
module org.egothor.radixor;
```
A named consuming module uses:
```java
module example.consumer {
requires org.egothor.radixor;
}
```
The core module is standalone and can be consumed directly as a normal Java module.
## Recommended reading order ## Recommended reading order
For most developers, the best order is: For most developers, the best order is:

View File

@@ -58,6 +58,27 @@ A deterministic system is easier to test, easier to reason about, and safer to i
The project is intended to maintain very high confidence in both core correctness and behavioral stability. The project is intended to maintain very high confidence in both core correctness and behavioral stability.
The recommended execution strategy is defined by the tagged test profiles in [Test taxonomy and execution filtering](test-taxonomy-and-filtering.md). In practice, teams can execute profile tasks directly:
- `./gradlew ciSmoke`: fast local/PR safety checks (`unit`, excluding `slow`; additionally excludes
`CompileIntegrationTest` as a defensive safeguard).
- `./gradlew ciSlow`: enterprise heavy gate for all tests marked with `slow` (typically
production dictionary and large corpus verification). This should be used for scheduled/manual
hardening gates and not in standard release build.
- `./gradlew ciCore`: behavioral coverage of trie and frequency-trie paths (`unit` + `property` where applicable)
- `./gradlew ciIntegration`: pipeline and CLI integration path checks
- `./gradlew ciCompat`: compatibility and regression verification for persisted artifacts
- `./gradlew ciRelease`: full non-slow suite for release-confidence runs (all test tags except `slow`,
plus explicit name-based exclusion of `CompileIntegrationTest*` and
`StemmerPatchTrieLoaderTest$BundledDictionaryTests*` as additional guardrails)
- `./gradlew ciNightly`: extended fuzz profile for robustness hardening
- `./gradlew ci`: umbrella profile depending on smoke/core/integration/compat
## Test taxonomy and execution filtering
The full tag taxonomy and executable filter examples are documented in
[Test taxonomy and execution filtering](test-taxonomy-and-filtering.md).
### Structural coverage ### Structural coverage
High code coverage is treated as a useful signal, but not as a sufficient goal on its own. Coverage is valuable only when the covered scenarios actually pressure the implementation in meaningful ways. High code coverage is treated as a useful signal, but not as a sufficient goal on its own. Coverage is valuable only when the covered scenarios actually pressure the implementation in meaningful ways.

View File

@@ -67,6 +67,36 @@ public final class LoadBinaryStemmerExample {
} }
``` ```
You can tune in-memory child lookup density at load time without changing the artifact:
```java
import java.io.IOException;
import java.nio.file.Path;
import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.StemmerPatchTrieLoader;
public final class LoadBinaryStemmerExampleTuned {
private LoadBinaryStemmerExampleTuned() {
throw new AssertionError("No instances.");
}
public static void main(final String[] arguments) throws IOException {
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"),
1024);
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
Path.of("stemmers", "english.radixor.gz"),
128);
System.out.println("fast=" + fast.size() + ", compact=" + compact.size());
}
}
```
For the trade-off details, see [Lookup Edge Optimization](lookup-edge-optimization.md).
### Build or extend a stemmer from dictionary data ### Build or extend a stemmer from dictionary data
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace. Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.

View File

@@ -23,7 +23,7 @@ These reports are primarily useful when reviewing the published API surface and
These reports describe the outcome of core verification and static-analysis stages for the latest published build: These reports describe the outcome of core verification and static-analysis stages for the latest published build:
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/) - [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html) - [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/) - [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/) - [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)

View File

@@ -0,0 +1,216 @@
# Test Tag Taxonomy and Execution Guide
Radixor uses JUnit tags as an explicit execution policy for its test suite.
The project uses three orthogonal axes:
1. **Scope** (how the test is executed in the pipeline)
2. **Domain** (where in the system it belongs)
3. **Intent** (what behavior it verifies)
## Canonical scope tags
| Tag | Description | Typical usage |
| --- | --- | --- |
| `unit` | Fast, deterministic tests that exercise a specific class or behavior without external processes. | Default developer feedback; should stay near-zero flakiness and low run time. |
| `integration` | Tests that span multiple components or end-to-end flows of the public pipeline. | Parser/loader/CLI/IO integration checks and multi-step compile-then-load validations. |
| `property` | Property-based tests with generator-driven coverage for invariants. | Semantics-preserving laws and edge-case exploration beyond curated fixtures. |
| `fuzz` | Randomized stress checks with bounded runtime. | Heavier probabilistic verification of robustness and reduction invariants. |
| `compat` | Backward/forward compatibility and reproducibility checks for persisted artifacts. | Artifact fingerprints, deterministic rebuild, and regression fixtures. |
| `slow` | Long-running or expensive tests that should not execute in every fast gate. | Heavy fuzz/property budgets or high-duration integration checks. |
## Canonical domain tags
| Tag | Description | Typical usage |
| --- | --- | --- |
| `core` | Core algorithm and foundational platform behavior. | Traversal direction, base data structures, low-level helpers. |
| `trie` | All mutable/compiled trie behaviors and traversal internals. | Lookup path selection, node shape, child representation, subtree behavior. |
| `frequency-trie` | Algorithms and corner cases specific to frequency-aware trie logic. | Ranking, weighted reductions, persistence of weighted nodes. |
| `stemmer` | End-user stemming pipeline semantics. | Parse-encode-apply flows and output invariants. |
| `patch` | Patch encoding, decoding, and application semantics. | `PatchCommandEncoder` behavior and related compatibility contracts. |
| `io` | Input/output and resource loading boundaries. | Filesystem readers, streams, and stream lifecycle handling. |
| `serialization` | Binary persistence contract of compiled artifacts. | Versioned format reads/writes and checksum/consistency checks. |
| `parser` | Dictionary and metadata parsing concerns. | Dictionary input parsing and malformed-source rejection. |
| `cli` | Command-line entrypoint and command orchestration behavior. | Compile CLI integration and CLI argument validation. |
| `metadata` | Trie metadata semantics, compatibility fields, and schema expectations. | Version flags, structural properties, and metadata round-trips. |
| `compile` | Compile-time pipeline and build-oriented behavior. | Building, reduction-mode behavior, and compiled artifact generation. |
| `diacritic` | Unicode diacritic normalization and stripping behavior. | Accent-removal correctness and locale-safe normalization checks. |
## Canonical intent tags
| Tag | Description | Typical usage |
| --- | --- | --- |
| `construction` | Tests around construction and assembly of runtime structures. | Builders, loaders, and compile-time object construction contracts. |
| `lookup` | Read behavior and retrieval semantics. | `get()`, `getAll()`, traversal and missing-key behavior. |
| `persistence` | Storage lifecycle semantics. | Serialization/deserialization and round-trip correctness. |
| `reduction` | Reduction algorithm correctness and corner cases. | Dominance threshold, subtree deduplication, rank-preservation invariants. |
| `encoding` | Encoding transformation direction. | `PatchCommandEncoder.encode` and serialized command form generation. |
| `decoding` | Decoding/interpretation of persisted or runtime commands. | Optional consumers that parse and apply encoded command payloads. |
| `apply` | Patch application and transformation behavior. | Verifies that applied patches produce expected derived forms. |
| `normalization` | Canonicalization and cleanup behavior. | String normalization around case/shape and mirrored input paths. |
| `validation` | Input rejection and defensive checks. | Null/empty/invalid contracts and explicit failure conditions. |
| `regression` | Guard tests for behavior changes over time. | Known historical bugs and behavioral drift prevention. |
| `determinism` | Repeatable results under fixed input and settings. | Compile determinism, stable ordering, and artifact reproducibility. |
| `error-handling` | Exception surface and robustness expectations. | Recovery/failure modes and diagnostics quality. |
## Class-level rules
1. Every test class has **exactly one** scope tag.
2. Every test class has at least one domain tag.
3. Additional tags describe intent and may be used on classes or nested tests.
4. For each test class, intent tags should reflect the primary behavior under test, not historical naming conventions.
## Governance and execution policy
The following rules are used to keep the suite auditable and stable:
| Rule | Required state | Why |
| --- | --- | --- |
| Scope discipline | Exactly one scope tag per class. | Prevents accidental promotion of integration-only behavior into fast unit runs. |
| Coverage breadth | At least one domain tag per class. | Ensures tests can be grouped by subsystem for targeted review. |
| Intent specificity | Use at least one intent tag when behavior is non-trivial. | Makes failure triage faster and profile composition explicit. |
| Runtime policy | Never run `slow` tests in the default `unit` profile unless explicitly required. | Preserves turnaround for PR feedback while preserving deep checks. |
| Change risk | Any persistence or compatibility-affecting change must include `compat` in validation. | Protects long-lived binary artifact contracts. |
| Mutation resistance | `fuzz`/`property` sets should be gated to dedicated profiles. | Limits flakiness exposure and controls CI resource cost. |
## Suggested CI profiles
These are recommended launch profiles for local and CI usage and are also exposed as Gradle tasks:
- **Profile: `ci-smoke` (fast feedback):**
```
./gradlew test -DincludeTags=unit -DexcludeTags=slow
./gradlew ciSmoke
```
`ciSmoke` also excludes `org.egothor.stemmer.CompileIntegrationTest*` at test-name filter level as a
defensive fallback in case of future tag drift.
`ciRelease` also excludes
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` at filter level.
- **Profile: `ci-core` (core behavioral coverage):**
```
./gradlew test -DincludeTags=unit,trie,frequency-trie,property
./gradlew ciCore
```
- **Profile: `ci-integration` (pipeline correctness):**
```
./gradlew test -DincludeTags=integration
./gradlew ciIntegration
```
- **Profile: `ci-slow` (explicit heavy validation):**
```
./gradlew ciSlow
```
- **Profile: `ci-compat` (artifact stability):**
```
./gradlew test -DincludeTags=compat,regression
./gradlew ciCompat
```
- **Profile: `ci-release` (strong confidence before release):**
```
./gradlew test -DexcludeTags=slow
./gradlew ciRelease
```
`ciRelease` is non-slow by policy and uses the same defensive name-based exclusion for
`org.egothor.stemmer.CompileIntegrationTest*` and
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` in addition to tag filtering.
- **Profile: `ci-nightly` (extended hardening):**
```
./gradlew test -DincludeTags=fuzz
./gradlew ciNightly
```
- **Profile: `ci` (enterprise umbrella):**
```
./gradlew ci
```
`ci` and `ciRelease` intentionally do **not** include `slow` paths. Run `ciSlow` explicitly for production-dictionary stress and long-running corpus checks.
## Practical examples
All examples use Gradle with JUnit Platform integration:
- Only unit tests:
```
./gradlew test -DincludeTags=unit
```
- Integration tests only:
```
./gradlew test -DincludeTags=integration
```
- Only trie subsystem tests:
```
./gradlew test -DincludeTags=trie
```
- Deterministic fuzz checks:
```
./gradlew test -DincludeTags=fuzz
```
- Property tests:
```
./gradlew test -DincludeTags=property
```
- Stemmer + patch command behavior:
```
./gradlew test -DincludeTags=stemmer,patch
```
- Compatibility artifacts and regression checks:
```
./gradlew test -DincludeTags=compat
```
- Keep regression suite and remove long-running cases:
```
./gradlew test -DincludeTags=regression -DexcludeTags=slow
```
- Core + patch behavior:
```
./gradlew test -DincludeTags=trie,patch
```
- Deterministic compatibility and persistence checks:
```
./gradlew test -DincludeTags=compat,determinism,serialization
```
## Notes
- `-DincludeTags` and `-DexcludeTags` are interpreted by Gradle task filtering and forwarded into
JUnit tag filtering.
- Class-name filtering is also available via Gradle test selectors where needed
(for example, `--tests *CompileTest`), but tag filtering remains the default
execution strategy.
- `-DincludeTags` supports comma-separated literal tags. When you need a single exact tag with special
characters, quote the argument for the shell.

View File

@@ -84,7 +84,7 @@ publishing {
} }
signing { signing {
required { !version.toString().endsWith('-SNAPSHOT') } required = !version.toString().endsWith('-SNAPSHOT')
if (signingKey != null && !signingKey.isBlank()) { if (signingKey != null && !signingKey.isBlank()) {
useInMemoryPgpKeys(signingKey, signingPassword) useInMemoryPgpKeys(signingKey, signingPassword)
sign publishing.publications.mavenJava sign publishing.publications.mavenJava

View File

@@ -54,6 +54,7 @@ nav:
- Overview: architecture-and-reduction.md - Overview: architecture-and-reduction.md
- Architecture: architecture.md - Architecture: architecture.md
- Reduction Semantics: reduction-semantics.md - Reduction Semantics: reduction-semantics.md
- Lookup Edge Optimization: lookup-edge-optimization.md
- Compatibility and Guarantees: compatibility-and-guarantees.md - Compatibility and Guarantees: compatibility-and-guarantees.md
- Dictionaries: - Dictionaries:
@@ -63,3 +64,4 @@ nav:
- Quality and Operations: quality-and-operations.md - Quality and Operations: quality-and-operations.md
- Benchmarking: benchmarking.md - Benchmarking: benchmarking.md
- Reports: reports.md - Reports: reports.md
- Test taxonomy and execution filtering: test-taxonomy-and-filtering.md

View File

@@ -31,11 +31,13 @@
package org.egothor.stemmer.benchmark; package org.egothor.stemmer.benchmark;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.egothor.stemmer.FrequencyTrie; import org.egothor.stemmer.FrequencyTrie;
import org.egothor.stemmer.PatchCommandEncoder; import org.egothor.stemmer.PatchCommandEncoder;
import org.egothor.stemmer.ReductionMode; import org.egothor.stemmer.ReductionMode;
import org.egothor.stemmer.ReductionSettings; import org.egothor.stemmer.ReductionSettings;
import org.egothor.stemmer.ValueCount;
import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Level;
@@ -97,12 +99,45 @@ public class FrequencyTrieLookupBenchmark {
*/ */
private String[] lookupKeys; private String[] lookupKeys;
/**
* Lookup keys as normalized caller-owned character storage.
*/
private char[][] lookupKeyCharacters;
/** /**
* Keys that are known to return multiple patch candidates from * Keys that are known to return multiple patch candidates from
* {@code getAll()}. * {@code getAll()}.
*/ */
private String[] ambiguousLookupKeys; private String[] ambiguousLookupKeys;
/**
* Ambiguous lookup keys as normalized caller-owned character storage.
*/
private char[][] ambiguousLookupKeyCharacters;
/**
* Preferred patches aligned with {@link #lookupKeys}.
*/
private String[] preferredPatches;
/**
* Reusable output buffer for patch application benchmarks.
*/
private char[] outputBuffer;
/**
* Mutable field consumed by visitor sinks.
*/
private int visitorAccumulator;
/**
* Sink used by visitor lookup benchmarks without per-invocation allocation.
*/
private final FrequencyTrie.EntrySink<String> visitorSink = (value, count, rank) -> {
this.visitorAccumulator += value.length() + count + rank;
return true;
};
/** /**
* Initializes the benchmark state. * Initializes the benchmark state.
* *
@@ -116,6 +151,23 @@ public class FrequencyTrieLookupBenchmark {
this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true); this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true);
this.lookupKeys = corpus.lookupKeys(); this.lookupKeys = corpus.lookupKeys();
this.ambiguousLookupKeys = corpus.ambiguousLookupKeys(); this.ambiguousLookupKeys = corpus.ambiguousLookupKeys();
this.lookupKeyCharacters = toCharArrays(this.lookupKeys);
this.ambiguousLookupKeyCharacters = toCharArrays(this.ambiguousLookupKeys);
this.preferredPatches = new String[this.lookupKeys.length];
int maxKeyLength = 0;
for (int index = 0; index < this.lookupKeys.length; index++) {
this.preferredPatches[index] = this.trie.get(this.lookupKeys[index]);
maxKeyLength = Math.max(maxKeyLength, this.lookupKeys[index].length());
}
this.outputBuffer = new char[maxKeyLength + 32];
}
private static char[][] toCharArrays(final String[] values) {
final char[][] characters = new char[values.length][];
for (int index = 0; index < values.length; index++) {
characters[index] = values[index].toCharArray();
}
return characters;
} }
} }
@@ -155,6 +207,61 @@ public class FrequencyTrieLookupBenchmark {
} }
} }
/**
* Measures retrieval of all patch candidates through caller-owned normalized
* character storage and a visitor sink.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupAllPatchesWithNormalizedCharVisitor(final LookupState state, final Blackhole blackhole) {
final char[][] keys = state.ambiguousLookupKeyCharacters;
for (char[] key : keys) {
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
if (count < 2) {
throw new IllegalStateException("Expected multiple patches for benchmark key.");
}
}
blackhole.consume(state.visitorAccumulator);
}
/**
* Measures counted candidate retrieval through the allocating entry API.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupPatchEntries(final LookupState state, final Blackhole blackhole) {
final String[] keys = state.ambiguousLookupKeys;
for (String key : keys) {
final List<ValueCount<String>> entries = state.trie.getEntries(key);
if (entries.size() < 2) {
throw new IllegalStateException("Expected multiple entries for key " + key + '.');
}
blackhole.consume(entries);
}
}
/**
* Measures counted candidate retrieval through the visitor API.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void lookupPatchEntriesWithVisitor(final LookupState state, final Blackhole blackhole) {
final char[][] keys = state.ambiguousLookupKeyCharacters;
for (char[] key : keys) {
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
if (count < 2) {
throw new IllegalStateException("Expected multiple entries for benchmark key.");
}
}
blackhole.consume(state.visitorAccumulator);
}
/** /**
* Measures end-to-end preferred stemming from lookup plus patch application. * Measures end-to-end preferred stemming from lookup plus patch application.
* *
@@ -170,6 +277,28 @@ public class FrequencyTrieLookupBenchmark {
} }
} }
/**
* Measures patch application into caller-owned output storage.
*
* @param state prepared lookup state
* @param blackhole sink preventing dead-code elimination
*/
@Benchmark
public void applyPreferredPatchToBuffer(final LookupState state, final Blackhole blackhole) {
final String[] keys = state.lookupKeys;
final String[] patches = state.preferredPatches;
final char[] output = state.outputBuffer;
for (int index = 0; index < keys.length; index++) {
final int length = PatchCommandEncoder.applyTo(keys[index], patches[index],
state.trie.traversalDirection(), output, 0, output.length);
if (length == PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
throw new IllegalStateException("Output buffer too small for key " + keys[index] + '.');
}
blackhole.consume(length);
blackhole.consume(output[0]);
}
}
/** /**
* Measures end-to-end full candidate stemming from {@code getAll()} plus * Measures end-to-end full candidate stemming from {@code getAll()} plus
* patch application. * patch application.

View File

@@ -0,0 +1,5 @@
module org.egothor.radixor {
requires java.logging;
exports org.egothor.stemmer;
}

View File

@@ -48,8 +48,7 @@ public enum CaseProcessingMode {
AS_IS, AS_IS,
/** /**
* Normalizes all dictionary content to lower case using * Normalizes all dictionary content to lower case using {@link Locale#ROOT}.
* {@link Locale#ROOT}.
*/ */
LOWERCASE_WITH_LOCALE_ROOT LOWERCASE_WITH_LOCALE_ROOT
} }

View File

@@ -93,12 +93,12 @@ final class DiacriticStripper {
} }
/** /**
* Removes supported diacritic marks and common Latin ligatures from the supplied * Removes supported diacritic marks and common Latin ligatures from the
* text. * supplied text.
* *
* <p> * <p>
* The method returns the original {@link String} instance when no replacement is * The method returns the original {@link String} instance when no replacement
* required, avoiding an unnecessary allocation on the common ASCII path. * is required, avoiding an unnecessary allocation on the common ASCII path.
* </p> * </p>
* *
* @param input text to normalize * @param input text to normalize

File diff suppressed because it is too large Load Diff

View File

@@ -119,11 +119,11 @@ public final class FrequencyTrieBuilders {
* Copies one compiled node and all reachable descendants into the target * Copies one compiled node and all reachable descendants into the target
* builder. * builder.
* *
* @param node current compiled node * @param node current compiled node
* @param keyBuilder current key builder * @param keyBuilder current key builder
* @param builder target mutable builder * @param builder target mutable builder
* @param traversalDirection logical key traversal direction used by the source * @param traversalDirection logical key traversal direction used by the source
* @param <V> value type * @param <V> value type
*/ */
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder, private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) { final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {

View File

@@ -67,7 +67,7 @@ import java.util.concurrent.locks.ReentrantLock;
* instance can still be used safely when needed. * instance can still be used safely when needed.
* </p> * </p>
*/ */
@SuppressWarnings("PMD.CyclomaticComplexity") @SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" })
public final class PatchCommandEncoder { public final class PatchCommandEncoder {
/** /**
@@ -121,6 +121,13 @@ public final class PatchCommandEncoder {
*/ */
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT }); /* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
/**
* Return value used by
* {@link #applyTo(CharSequence, String, WordTraversalDirection, char[], int, int)}
* when the caller-owned output range is too small for the transformed text.
*/
public static final int APPLY_INSUFFICIENT_CAPACITY = -1;
/** /**
* Prefix used in unsupported NOOP patch argument exceptions. * Prefix used in unsupported NOOP patch argument exceptions.
*/ */
@@ -346,6 +353,78 @@ public final class PatchCommandEncoder {
return applyStrategyFor(traversalDirection).apply(source, patchCommand); return applyStrategyFor(traversalDirection).apply(source, patchCommand);
} }
/**
* Applies a compact patch command into a caller-owned output buffer.
*
* <p>
* The output array is not retained. Capacity failure is reported by
* {@link #APPLY_INSUFFICIENT_CAPACITY} and leaves the output range unchanged.
* Malformed compatibility cases preserve the source exactly as
* {@link #apply(String, String, WordTraversalDirection)} does.
* </p>
*
* @param source original source text
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @param output caller-owned output storage
* @param outputOffset first writable output offset
* @param outputLength writable output capacity
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
* when {@code outputLength} is too small
*/
public static int applyTo(final CharSequence source, final String patchCommand,
final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset,
final int outputLength) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(traversalDirection, "traversalDirection");
Objects.requireNonNull(output, "output");
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
final int sourceLength = source.length();
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
if (producedLength > outputLength) {
return APPLY_INSUFFICIENT_CAPACITY;
}
applyToOutput(source, 0, sourceLength, patchCommand, traversalDirection, output, outputOffset, producedLength);
return producedLength;
}
/**
* Applies a compact patch command from a caller-owned source slice into a
* caller-owned output buffer.
*
* @param source source storage
* @param sourceOffset first source character offset
* @param sourceLength number of source characters
* @param patchCommand compact patch command
* @param traversalDirection traversal direction used by the patch command
* @param output caller-owned output storage
* @param outputOffset first writable output offset
* @param outputLength writable output capacity
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
* when {@code outputLength} is too small
* @throws IllegalArgumentException when source and output ranges overlap in the
* same array
*/
public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int outputLength) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(traversalDirection, "traversalDirection");
Objects.requireNonNull(output, "output");
Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length);
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength);
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
if (producedLength > outputLength) {
return APPLY_INSUFFICIENT_CAPACITY;
}
applyToOutput(source, sourceOffset, sourceLength, patchCommand, traversalDirection, output, outputOffset,
producedLength);
return producedLength;
}
/** /**
* Encodes a patch command using the historical backward Egothor semantics. * Encodes a patch command using the historical backward Egothor semantics.
* *
@@ -409,7 +488,6 @@ public final class PatchCommandEncoder {
* @param patchCommand compact patch command * @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null} * @return transformed word, or {@code null} when {@code source} is {@code null}
*/ */
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
private static String applyBackward(final String source, final String patchCommand) { private static String applyBackward(final String source, final String patchCommand) {
if (source == null) { if (source == null) {
return null; return null;
@@ -435,7 +513,7 @@ public final class PatchCommandEncoder {
int position = result.length() - 1; int position = result.length() - 1;
try { try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex); final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1); final char argument = patchCommand.charAt(patchIndex + 1);
@@ -493,7 +571,6 @@ public final class PatchCommandEncoder {
* @param patchCommand compact patch command * @param patchCommand compact patch command
* @return transformed word, or {@code null} when {@code source} is {@code null} * @return transformed word, or {@code null} when {@code source} is {@code null}
*/ */
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
private static String applyForward(final String source, final String patchCommand) { private static String applyForward(final String source, final String patchCommand) {
if (source == null) { if (source == null) {
return null; return null;
@@ -519,7 +596,7 @@ public final class PatchCommandEncoder {
int position = 0; int position = 0;
try { try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex); final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1); final char argument = patchCommand.charAt(patchIndex + 1);
@@ -681,7 +758,7 @@ public final class PatchCommandEncoder {
*/ */
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) { private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
try { try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex); final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1); final char argument = patchCommand.charAt(patchIndex + 1);
@@ -722,7 +799,7 @@ public final class PatchCommandEncoder {
*/ */
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) { private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
try { try {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex); final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1); final char argument = patchCommand.charAt(patchIndex + 1);
@@ -753,6 +830,711 @@ public final class PatchCommandEncoder {
return result.toString(); return result.toString();
} }
/**
* Computes the transformed length or the preserved source length for malformed
* compatibility cases.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return produced length
*/
private static int computeAppliedLength(final int sourceLength, final String patchCommand,
final WordTraversalDirection traversalDirection) {
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|| (patchCommand.length() & 1) != 0) {
return sourceLength;
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
return computeBackwardAppliedLength(sourceLength, patchCommand);
}
return computeForwardAppliedLength(sourceLength, patchCommand);
}
/**
* Computes the backward traversal output length.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return produced length
*/
private static int computeBackwardAppliedLength(final int sourceLength, final String patchCommand) {
if (patchCommand.length() == 2) {
return computeSingleBackwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return computeBackwardEmptyAppliedLength(patchCommand);
}
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return sourceLength;
}
position = position - skipCount + 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return sourceLength;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1) {
return sourceLength;
}
final int deleteEndExclusive = position + 1;
position -= deleteCount - 1;
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
return sourceLength;
}
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
if (position < -1 || position >= currentLength) {
return sourceLength;
}
currentLength++;
position++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position--;
}
return currentLength;
}
/**
* Computes the forward traversal output length.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return produced length
*/
private static int computeForwardAppliedLength(final int sourceLength, final String patchCommand) {
if (patchCommand.length() == 2) {
return computeSingleForwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return computeForwardEmptyAppliedLength(patchCommand);
}
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return sourceLength;
}
position = position + skipCount - 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return sourceLength;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
return sourceLength;
}
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
if (position < 0 || position > currentLength) {
return sourceLength;
}
currentLength++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position++;
}
return currentLength;
}
/**
* Computes a single backward instruction output length.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return produced length
*/
private static int computeSingleBackwardAppliedLength(final int sourceLength, final char opcode,
final char argument) {
final int encodedValue;
switch (opcode) {
case DELETE_OPCODE:
encodedValue = decodeEncodedCount(argument);
return encodedValue < 1 || encodedValue > sourceLength ? sourceLength : sourceLength - encodedValue;
case INSERT_OPCODE:
return sourceLength + 1;
case REPLACE_OPCODE:
case SKIP_OPCODE:
return sourceLength;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return sourceLength;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
/**
* Computes a single forward instruction output length.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return produced length
*/
private static int computeSingleForwardAppliedLength(final int sourceLength, final char opcode,
final char argument) {
return computeSingleBackwardAppliedLength(sourceLength, opcode, argument);
}
/**
* Computes output length for an empty source in backward traversal.
*
* @param patchCommand patch command
* @return produced length
*/
private static int computeBackwardEmptyAppliedLength(final String patchCommand) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case INSERT_OPCODE:
currentLength++;
break;
case SKIP_OPCODE:
case REPLACE_OPCODE:
case DELETE_OPCODE:
return 0;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return 0;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
return currentLength;
}
/**
* Computes output length for an empty source in forward traversal.
*
* @param patchCommand patch command
* @return produced length
*/
private static int computeForwardEmptyAppliedLength(final String patchCommand) {
return computeBackwardEmptyAppliedLength(patchCommand);
}
/**
* Applies an already-sized patch into caller output.
*
* @param source source text
* @param sourceOffset source offset
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @param output output storage
* @param outputOffset output offset
* @param producedLength already-validated produced length
*/
private static void applyToOutput(final CharSequence source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int producedLength) {
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
copySource(source, sourceOffset, sourceLength, output, outputOffset);
return;
}
if (sourceLength > 0) {
copySource(source, sourceOffset, sourceLength, output, outputOffset);
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
} else {
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
}
}
/**
* Applies an already-sized patch into caller output.
*
* @param source source storage
* @param sourceOffset source offset
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @param output output storage
* @param outputOffset output offset
* @param producedLength already-validated produced length
*/
private static void applyToOutput(final char[] source, final int sourceOffset, final int sourceLength,
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
final int outputOffset, final int producedLength) {
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
return;
}
if (sourceLength > 0) {
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
}
if (traversalDirection == WordTraversalDirection.BACKWARD) {
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
} else {
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
}
}
/**
* Determines whether the output is exactly the original source.
*
* @param sourceLength source length
* @param producedLength produced length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return {@code true} if copying the source is sufficient
*/
private static boolean isPreservedSource(final int sourceLength, final int producedLength,
final String patchCommand, final WordTraversalDirection traversalDirection) {
return producedLength == sourceLength
&& isKnownPreserveOnlyPatch(sourceLength, patchCommand, traversalDirection);
}
/**
* Returns whether equal length also means no mutation is needed.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param traversalDirection traversal direction
* @return {@code true} when the command preserves source content
*/
private static boolean isKnownPreserveOnlyPatch(final int sourceLength, final String patchCommand,
final WordTraversalDirection traversalDirection) {
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|| (patchCommand.length() & 1) != 0) {
return true;
}
if (patchCommand.length() == 2) {
return isSingleInstructionPreserveOnly(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
}
if (sourceLength == 0) {
return hasEmptySourcePreserveOnlyPatch(patchCommand);
}
return traversalDirection == WordTraversalDirection.BACKWARD
? hasBackwardPreserveOnlyPatch(sourceLength, patchCommand)
: hasForwardPreserveOnlyPatch(sourceLength, patchCommand);
}
/**
* Tests whether a single instruction preserves the source content.
*
* @param sourceLength source length
* @param opcode opcode
* @param argument argument
* @return {@code true} when no mutation should be applied
*/
private static boolean isSingleInstructionPreserveOnly(final int sourceLength, final char opcode,
final char argument) {
switch (opcode) {
case DELETE_OPCODE:
final int encodedValue = decodeEncodedCount(argument);
return encodedValue < 1 || encodedValue > sourceLength;
case INSERT_OPCODE:
return false;
case REPLACE_OPCODE:
return sourceLength == 0;
case SKIP_OPCODE:
return true;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
/**
* Tests whether an empty-source patch preserves the source.
*
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasEmptySourcePreserveOnlyPatch(final String patchCommand) {
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case INSERT_OPCODE:
break;
case SKIP_OPCODE:
case REPLACE_OPCODE:
case DELETE_OPCODE:
return true;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
}
return false;
}
/**
* Tests whether a backward patch preserves the source because it is malformed
* or a NOOP.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasBackwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return true;
}
position = position - skipCount + 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return true;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1) {
return true;
}
final int deleteEndExclusive = position + 1;
position -= deleteCount - 1;
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
return true;
}
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
if (position < -1 || position >= currentLength) {
return true;
}
currentLength++;
position++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position--;
}
return false;
}
/**
* Tests whether a forward patch preserves the source because it is malformed or
* a NOOP.
*
* @param sourceLength source length
* @param patchCommand patch command
* @return {@code true} when no mutation should be applied
*/
private static boolean hasForwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
final int skipCount = decodeEncodedCount(argument);
if (skipCount < 1) {
return true;
}
position = position + skipCount - 1;
break;
case REPLACE_OPCODE:
if (position < 0 || position >= currentLength) {
return true;
}
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
return true;
}
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
if (position < 0 || position > currentLength) {
return true;
}
currentLength++;
break;
case NOOP_OPCODE:
if (argument != NOOP_ARGUMENT) {
throw new IllegalArgumentException(MSG_NOOP + argument);
}
return true;
default:
throw new IllegalArgumentException(MSG_OPCODE + opcode);
}
position++;
}
return false;
}
/**
* Copies source characters from a sequence.
*
* @param source source text
* @param sourceOffset source offset
* @param sourceLength source length
* @param output output storage
* @param outputOffset output offset
*/
private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength,
final char[] output, final int outputOffset) {
for (int index = 0; index < sourceLength; index++) {
output[outputOffset + index] = source.charAt(sourceOffset + index);
}
}
/**
* Applies a backward patch after validation.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param output output storage initialized with source
* @param outputOffset output offset
*/
private static void applyBackwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
final int outputOffset) {
if (sourceLength == 0) {
applyBackwardEmptyToOutput(patchCommand, output, outputOffset);
return;
}
int currentLength = sourceLength;
int position = sourceLength - 1;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
position = position - decodeEncodedCount(argument) + 1;
break;
case REPLACE_OPCODE:
output[outputOffset + position] = argument;
break;
case DELETE_OPCODE:
final int deleteEndExclusive = position + 1;
position -= decodeEncodedCount(argument) - 1;
System.arraycopy(output, outputOffset + deleteEndExclusive, output, outputOffset + position,
currentLength - deleteEndExclusive);
currentLength -= deleteEndExclusive - position;
break;
case INSERT_OPCODE:
final int insertIndex = position + 1;
System.arraycopy(output, outputOffset + insertIndex, output, outputOffset + insertIndex + 1,
currentLength - insertIndex);
output[outputOffset + insertIndex] = argument;
currentLength++;
position++;
break;
case NOOP_OPCODE:
return;
default:
throw new AssertionError("Patch command was not validated.");
}
position--;
}
}
/**
* Applies a forward patch after validation.
*
* @param sourceLength source length
* @param patchCommand patch command
* @param output output storage initialized with source
* @param outputOffset output offset
*/
private static void applyForwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
final int outputOffset) {
if (sourceLength == 0) {
applyForwardEmptyToOutput(patchCommand, output, outputOffset);
return;
}
int currentLength = sourceLength;
int position = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char opcode = patchCommand.charAt(patchIndex);
final char argument = patchCommand.charAt(patchIndex + 1);
switch (opcode) {
case SKIP_OPCODE:
position = position + decodeEncodedCount(argument) - 1;
break;
case REPLACE_OPCODE:
output[outputOffset + position] = argument;
break;
case DELETE_OPCODE:
final int deleteCount = decodeEncodedCount(argument);
System.arraycopy(output, outputOffset + position + deleteCount, output, outputOffset + position,
currentLength - position - deleteCount);
currentLength -= deleteCount;
position--;
break;
case INSERT_OPCODE:
System.arraycopy(output, outputOffset + position, output, outputOffset + position + 1,
currentLength - position);
output[outputOffset + position] = argument;
currentLength++;
break;
case NOOP_OPCODE:
return;
default:
throw new AssertionError("Patch command was not validated.");
}
position++;
}
}
/**
* Applies an empty-source backward patch after validation.
*
* @param patchCommand patch command
* @param output output storage
* @param outputOffset output offset
*/
private static void applyBackwardEmptyToOutput(final String patchCommand, final char[] output,
final int outputOffset) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
final char argument = patchCommand.charAt(patchIndex + 1);
System.arraycopy(output, outputOffset, output, outputOffset + 1, currentLength);
output[outputOffset] = argument;
currentLength++;
}
}
/**
* Applies an empty-source forward patch after validation.
*
* @param patchCommand patch command
* @param output output storage
* @param outputOffset output offset
*/
private static void applyForwardEmptyToOutput(final String patchCommand, final char[] output,
final int outputOffset) {
int currentLength = 0;
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
output[outputOffset + currentLength] = patchCommand.charAt(patchIndex + 1);
currentLength++;
}
}
/**
* Validates that source and output slices do not overlap when backed by the
* same array.
*
* @param source source storage
* @param sourceOffset source offset
* @param sourceLength source length
* @param output output storage
* @param outputOffset output offset
* @param outputLength output length
*/
private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset,
final int sourceLength, final char[] output, final int outputOffset, final int outputLength) {
if (!source.equals(output) || sourceLength == 0 || outputLength == 0) {
return;
}
final int sourceEnd = sourceOffset + sourceLength;
final int outputEnd = outputOffset + outputLength;
if (sourceOffset < outputEnd && outputOffset < sourceEnd) {
throw new IllegalArgumentException("source and output ranges must not overlap.");
}
}
/** /**
* Returns the direction-specialized apply strategy. * Returns the direction-specialized apply strategy.
* *
@@ -769,7 +1551,6 @@ public final class PatchCommandEncoder {
* @param argument serialized count argument * @param argument serialized count argument
* @return decoded positive count, or {@code -1} when the argument is malformed * @return decoded positive count, or {@code -1} when the argument is malformed
*/ */
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
private static int decodeEncodedCount(final char argument) { private static int decodeEncodedCount(final char argument) {
if (argument < 'a') { if (argument < 'a') {
return -1; return -1;

View File

@@ -94,6 +94,29 @@ public final class StemmerPatchTrieBinaryIO {
} }
} }
/**
* Reads a GZip-compressed binary patch-command trie from a filesystem path with
* an optional dense child lookup span override.
* <p>
* This is a runtime-only tuning parameter. The dense-span setting is not
* persisted in the file and does not change the compiled metadata.
* </p>
*
* @param path source file
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return deserialized trie
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(path, "path");
try (InputStream fileInputStream = Files.newInputStream(path)) {
return read(fileInputStream, maxExpandedIndex);
}
}
/** /**
* Reads a GZip-compressed binary patch-command trie from a filesystem path * Reads a GZip-compressed binary patch-command trie from a filesystem path
* string. * string.
@@ -108,6 +131,26 @@ public final class StemmerPatchTrieBinaryIO {
return read(Path.of(fileName)); return read(Path.of(fileName));
} }
/**
* Reads a GZip-compressed binary patch-command trie from a filesystem path
* string with an optional dense child lookup span override.
* <p>
* This is a runtime-only tuning parameter. The dense-span setting is not
* persisted in the file and does not change the compiled metadata.
* </p>
*
* @param fileName source file name or path string
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return deserialized trie
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(fileName, "fileName");
return read(Path.of(fileName), maxExpandedIndex);
}
/** /**
* Reads a GZip-compressed binary patch-command trie from an input stream. * Reads a GZip-compressed binary patch-command trie from an input stream.
* *
@@ -132,6 +175,35 @@ public final class StemmerPatchTrieBinaryIO {
} }
} }
/**
* Reads a GZip-compressed binary patch-command trie from an input stream with
* an optional dense child lookup span override.
* <p>
* This is a runtime-only tuning parameter. The dense-span setting is not
* persisted in the file and does not change the compiled metadata.
* </p>
*
* @param inputStream source stream
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return deserialized trie
* @throws NullPointerException if {@code inputStream} is {@code null}
* @throws IOException if reading or decompression fails
*/
public static FrequencyTrie<String> read(final InputStream inputStream, final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(inputStream, "inputStream");
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));
DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) {
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(dataInputStream, String[]::new, STRING_CODEC,
maxExpandedIndex);
LOGGER.log(Level.FINE, "Read compressed binary stemmer trie.");
return trie;
}
}
/** /**
* Reads only metadata from a GZip-compressed binary patch-command trie stored * Reads only metadata from a GZip-compressed binary patch-command trie stored
* at a filesystem path. * at a filesystem path.

View File

@@ -71,6 +71,7 @@ import java.util.zip.GZIPInputStream;
public final class StemmerPatchTrieLoader { public final class StemmerPatchTrieLoader {
/* default */ static final String FILENAME_REQUIRED = "fileName required"; /* default */ static final String FILENAME_REQUIRED = "fileName required";
private static final String PARAMETER_PATH = "path";
/** /**
* Logger of this class. * Logger of this class.
@@ -461,7 +462,7 @@ public final class StemmerPatchTrieLoader {
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection, final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode) final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
throws IOException { throws IOException {
Objects.requireNonNull(path, "path"); Objects.requireNonNull(path, PARAMETER_PATH);
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode, final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
diacriticProcessingMode); diacriticProcessingMode);
return load(path, storeOriginal, metadata); return load(path, storeOriginal, metadata);
@@ -487,7 +488,7 @@ public final class StemmerPatchTrieLoader {
*/ */
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata) public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
throws IOException { throws IOException {
Objects.requireNonNull(path, "path"); Objects.requireNonNull(path, PARAMETER_PATH);
Objects.requireNonNull(metadata, "metadata"); Objects.requireNonNull(metadata, "metadata");
try (InputStream inputStream = openDictionaryInputStream(path); try (InputStream inputStream = openDictionaryInputStream(path);
@@ -759,10 +760,31 @@ public final class StemmerPatchTrieLoader {
* read * read
*/ */
public static FrequencyTrie<String> loadBinary(final Path path) throws IOException { public static FrequencyTrie<String> loadBinary(final Path path) throws IOException {
Objects.requireNonNull(path, "path"); Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.read(path); return StemmerPatchTrieBinaryIO.read(path);
} }
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* using a custom dense lookup span override.
* <p>
* This is a runtime-only tuning parameter that does not affect persisted
* metadata.
* </p>
*
* @param path path to the compressed binary trie file
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return compiled patch-command trie
* @throws NullPointerException if {@code path} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<String> loadBinary(final Path path, final int maxExpandedIndex) throws IOException {
Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex);
}
/** /**
* Loads a GZip-compressed binary patch-command trie from a filesystem path * Loads a GZip-compressed binary patch-command trie from a filesystem path
* string. * string.
@@ -778,6 +800,28 @@ public final class StemmerPatchTrieLoader {
return StemmerPatchTrieBinaryIO.read(fileName); return StemmerPatchTrieBinaryIO.read(fileName);
} }
/**
* Loads a GZip-compressed binary patch-command trie from a filesystem path
* string using a custom dense lookup span override.
* <p>
* This is a runtime-only tuning parameter that does not affect persisted
* metadata.
* </p>
*
* @param fileName file name or path string
* @param maxExpandedIndex dense lookup span override; negative values use
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
* @return compiled patch-command trie
* @throws NullPointerException if {@code fileName} is {@code null}
* @throws IOException if the file cannot be opened, decompressed, or
* read
*/
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex)
throws IOException {
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex);
}
/** /**
* Loads a GZip-compressed binary patch-command trie from an input stream. * Loads a GZip-compressed binary patch-command trie from an input stream.
* *
@@ -802,7 +846,7 @@ public final class StemmerPatchTrieLoader {
* read * read
*/ */
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException { public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
Objects.requireNonNull(path, "path"); Objects.requireNonNull(path, PARAMETER_PATH);
return StemmerPatchTrieBinaryIO.readMetadata(path); return StemmerPatchTrieBinaryIO.readMetadata(path);
} }
@@ -845,7 +889,7 @@ public final class StemmerPatchTrieLoader {
*/ */
public static void saveBinary(final FrequencyTrie<String> trie, final Path path) throws IOException { public static void saveBinary(final FrequencyTrie<String> trie, final Path path) throws IOException {
Objects.requireNonNull(trie, "trie"); Objects.requireNonNull(trie, "trie");
Objects.requireNonNull(path, "path"); Objects.requireNonNull(path, PARAMETER_PATH);
StemmerPatchTrieBinaryIO.write(trie, path); StemmerPatchTrieBinaryIO.write(trie, path);
} }

View File

@@ -58,17 +58,17 @@
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the * {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
* traditional line-oriented tab-separated values resource format in which each * traditional line-oriented tab-separated values resource format in which each
* non-empty logical line starts with a canonical stem followed by known surface * non-empty logical line starts with a canonical stem followed by known surface
* variants in subsequent tab-separated columns. * variants in subsequent tab-separated columns. Parsing is delegated to
* Parsing is delegated to {@link org.egothor.stemmer.StemmerDictionaryParser}, * {@link org.egothor.stemmer.StemmerDictionaryParser}, which applies
* which applies configurable case processing through * configurable case processing through
* {@link org.egothor.stemmer.CaseProcessingMode} (default: * {@link org.egothor.stemmer.CaseProcessingMode} (default:
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}), * {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
* supports whole-line as well as trailing remarks introduced by {@code #} or * supports whole-line as well as trailing remarks introduced by {@code #} or
* {@code //}, and currently ignores dictionary items containing Unicode * {@code //}, and currently ignores dictionary items containing Unicode
* whitespace characters while reporting them through warning-level diagnostics. * whitespace characters while reporting them through warning-level diagnostics.
* During loading, each variant is converted into a patch command * During loading, each variant is converted into a patch command targeting the
* targeting the canonical stem, and the stem itself may optionally be stored * canonical stem, and the stem itself may optionally be stored under the
* under the canonical no-operation patch. * canonical no-operation patch.
* </p> * </p>
* *
* <p> * <p>

View File

@@ -1,21 +1,21 @@
/******************************************************************************* /*******************************************************************************
* Copyright (C) 2026, Leo Galambos * Copyright (C) 2026, Leo Galambos
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
* *
* 1. Redistributions of source code must retain the above copyright notice, * 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer. * this list of conditions and the following disclaimer.
* *
* 2. Redistributions in binary form must reproduce the above copyright notice, * 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation * this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution. * and/or other materials provided with the distribution.
* *
* 3. Neither the name of the copyright holder nor the names of its contributors * 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software * may be used to endorse or promote products derived from this software
* without specific prior written permission. * without specific prior written permission.
* *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -43,14 +43,15 @@ import java.util.Objects;
* immutable from the public API perspective because construction wires these * immutable from the public API perspective because construction wires these
* arrays once and all lookup operations thereafter treat them as read-only. * arrays once and all lookup operations thereafter treat them as read-only.
* *
* @param <V> value type * @param <V> value type
* @param edgeLabels internal edge label array
* @param children internal child array
* @param orderedValues internal ordered values array
* @param orderedCounts internal ordered counts array
*/ */
@SuppressWarnings("PMD.DataClass") public final class CompiledNode<V> {
public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[] orderedValues, int... orderedCounts) {
/**
* Default dense child lookup span in characters used when an explicit override
* is not provided.
*/
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
/** /**
* Number of child edges where linear scan is cheaper than binary search. * Number of child edges where linear scan is cheaper than binary search.
@@ -58,24 +59,112 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4; private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4;
/** /**
* Creates one validated compiled node. * Edge labels in sorted ascending order.
*/
private final char[] edgeLabels;
/**
* Sparse child array aligned with {@link #edgeLabels}.
*/
private final CompiledNode<V>[] children;
/**
* Dense child lookup table used when labels fit into a compact char interval.
* <p>
* The table enables direct O(1) indexing for child lookup and is allocated only
* when the character span of this node's edges is within the configured
* threshold.
* </p>
*/
private final CompiledNode<V>[] denseChildren;
/**
* Normalized minimum edge value for the dense lookup table.
*/
private final int denseEdgeMin;
/**
* Values stored at this node in local order.
*/
private final V[] orderedValues;
/**
* Occurrence counts aligned with {@link #orderedValues}.
*/
private final int[] orderedCounts;
/**
* Creates one validated compiled node using {@link #DEFAULT_MAX_EXPANDED_INDEX}
* for dense lookup sizing.
* *
* @throws NullPointerException if any array argument is {@code null} * @throws NullPointerException if any array argument is {@code null}
* @throws IllegalArgumentException if the edge-related arrays or value-related * @throws IllegalArgumentException if the edge-related arrays or value-related
* arrays do not have matching lengths * arrays do not have matching lengths
*/ */
public CompiledNode { public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
final int... orderedCounts) {
this(edgeLabels, children, orderedValues, DEFAULT_MAX_EXPANDED_INDEX, orderedCounts);
}
/**
* Creates one validated compiled node.
*
* @param maxExpandedIndex upper bound for the dense lookup interval size; zero
* disables dense lookup. Larger values improve
* direct-index likelihood while increasing dense table
* memory in compact-label nodes.
* @throws NullPointerException if any array argument is {@code null}
* @throws IllegalArgumentException if the edge-related arrays or value-related
* arrays do not have matching lengths or the
* dense interval size is negative
*/
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
final int maxExpandedIndex, final int... orderedCounts) {
Objects.requireNonNull(edgeLabels, "edgeLabels"); Objects.requireNonNull(edgeLabels, "edgeLabels");
Objects.requireNonNull(children, "children"); Objects.requireNonNull(children, "children");
Objects.requireNonNull(orderedValues, "orderedValues"); Objects.requireNonNull(orderedValues, "orderedValues");
Objects.requireNonNull(orderedCounts, "orderedCounts"); Objects.requireNonNull(orderedCounts, "orderedCounts");
if (maxExpandedIndex < 0) {
throw new IllegalArgumentException("maxExpandedIndex must be non-negative.");
}
if (edgeLabels.length != children.length) { if (edgeLabels.length != children.length) {
throw new IllegalArgumentException("edgeLabels and children must have the same length."); throw new IllegalArgumentException("edgeLabels and children must have the same length.");
} }
if (orderedValues.length != orderedCounts.length) { if (orderedValues.length != orderedCounts.length) {
throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length."); throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length.");
} }
this.edgeLabels = edgeLabels;
this.children = children;
this.orderedValues = orderedValues;
this.orderedCounts = orderedCounts;
if (edgeLabels.length == 0 || maxExpandedIndex == 0) {
this.denseChildren = null;
this.denseEdgeMin = 0;
return;
}
final int minEdge = edgeLabels[0];
final int maxEdge = edgeLabels[edgeLabels.length - 1];
final int span = maxEdge - minEdge;
if (span < 0 || span > maxExpandedIndex) {
this.denseChildren = null;
this.denseEdgeMin = 0;
return;
}
@SuppressWarnings("unchecked")
final CompiledNode<V>[] dense = new CompiledNode[span + 1];
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
}
this.denseChildren = dense;
this.denseEdgeMin = minEdge;
} }
/** /**
@@ -87,7 +176,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
* *
* @return internal edge-label array * @return internal edge-label array
*/ */
@Override
@SuppressWarnings("PMD.MethodReturnsInternalArray") @SuppressWarnings("PMD.MethodReturnsInternalArray")
public char[] edgeLabels() { public char[] edgeLabels() {
return this.edgeLabels; return this.edgeLabels;
@@ -102,7 +190,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
* *
* @return internal child-node array * @return internal child-node array
*/ */
@Override
@SuppressWarnings("PMD.MethodReturnsInternalArray") @SuppressWarnings("PMD.MethodReturnsInternalArray")
public CompiledNode<V>[] children() { public CompiledNode<V>[] children() {
return this.children; return this.children;
@@ -117,7 +204,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
* *
* @return internal ordered-values array * @return internal ordered-values array
*/ */
@Override
@SuppressWarnings("PMD.MethodReturnsInternalArray") @SuppressWarnings("PMD.MethodReturnsInternalArray")
public V[] orderedValues() { public V[] orderedValues() {
return this.orderedValues; return this.orderedValues;
@@ -132,14 +218,143 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
* *
* @return internal ordered-counts array * @return internal ordered-counts array
*/ */
@Override
@SuppressWarnings("PMD.MethodReturnsInternalArray") @SuppressWarnings("PMD.MethodReturnsInternalArray")
public int[] orderedCounts() { public int[] orderedCounts() {
return this.orderedCounts; return this.orderedCounts;
} }
/**
* Returns the number of child edges represented by this node.
*
* @return child edge count
*/
public int edgeCount() {
return this.edgeLabels.length;
}
/**
* Returns the number of values stored in this node.
*
* @return value count
*/
public int valueCount() {
return this.orderedValues.length;
}
/**
* Indicates whether this node stores any values.
*
* @return {@code true} when values are present at this node
*/
public boolean hasValues() {
return this.orderedValues.length > 0;
}
/**
* Indicates whether this node has child edges.
*
* @return {@code true} when this node has at least one outgoing edge
*/
public boolean hasChildren() {
return this.edgeLabels.length > 0;
}
/**
* Indicates whether this node has no child edges.
*
* @return {@code true} when this node is a terminal leaf node
*/
public boolean isLeaf() {
return !hasChildren();
}
/**
* Tests whether an edge label is present at this node.
*
* @param edge edge label
* @return {@code true} if this node contains the supplied edge label
*/
public boolean hasEdge(final char edge) {
return findChild(edge) != null;
}
/**
* Indicates whether this node has a dense direct-index child lookup table.
*
* @return {@code true} when a direct-index child table is available
*/
public boolean hasDenseLookup() {
return this.denseChildren != null;
}
/**
* Returns a small memory-related metric describing this node's dense table
* size.
*
* @return number of dense table slots, or {@code 0} when dense lookup is not
* enabled
*/
public int denseTableLength() {
return this.denseChildren == null ? 0 : this.denseChildren.length;
}
/**
* Returns a compact structural summary used by diagnostics and tests.
*
* @return summary hash for node structure and contents
*/
@Override
public int hashCode() {
int hash = Arrays.hashCode(this.edgeLabels);
hash = 31 * hash + Arrays.hashCode(this.children);
hash = 31 * hash + Arrays.hashCode(this.orderedValues);
hash = 31 * hash + Arrays.hashCode(this.orderedCounts);
hash = 31 * hash + Objects.hash(this.denseEdgeMin);
hash = 31 * hash + (hasDenseLookup() ? Arrays.hashCode(this.denseChildren) : 0);
return hash;
}
/**
* Compares structural node content, including dense table availability.
*
* @param object comparison object
* @return {@code true} when nodes describe identical structure and payload
*/
@Override
public boolean equals(final Object object) {
if (this == object) {
return true;
}
if (!(object instanceof CompiledNode<?> other)) {
return false;
}
return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children)
&& Arrays.equals(this.orderedValues, other.orderedValues)
&& Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin
&& Arrays.equals(this.denseChildren, other.denseChildren);
}
/**
* Returns a short summary useful for debugging and diagnostics.
*
* @return textual node summary
*/
@Override
public String toString() {
return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount="
+ this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}';
}
/** /**
* Finds a child for the supplied edge character. * Finds a child for the supplied edge character.
*
* Lookup order is:
* <ol>
* <li>dense array index (if the label interval is compact enough),</li>
* <li>small-child linear scan when the fallback node has
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
* <li>binary search over sorted labels.</li>
* </ol>
* *
* @param edge edge character * @param edge edge character
* @return child node, or {@code null} if absent * @return child node, or {@code null} if absent
@@ -149,6 +364,15 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
if (childCount == 0) { if (childCount == 0) {
return null; return null;
} }
if (this.denseChildren != null) {
final int denseIndex = edge - this.denseEdgeMin;
if (denseIndex < 0 || denseIndex >= this.denseChildren.length) {
return null;
}
return this.denseChildren[denseIndex];
}
if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) { if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) {
for (int index = 0; index < childCount; index++) { for (int index = 0; index < childCount; index++) {
if (this.edgeLabels[index] == edge) { if (this.edgeLabels[index] == edge) {

View File

@@ -95,6 +95,9 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("integration") @Tag("integration")
@Tag("cli") @Tag("cli")
@Tag("stemmer") @Tag("stemmer")
@Tag("compile")
@Tag("construction")
@Tag("slow")
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
@DisplayName("Compile integration") @DisplayName("Compile integration")
final class CompileIntegrationTest { final class CompileIntegrationTest {
@@ -182,6 +185,11 @@ final class CompileIntegrationTest {
@Nested @Nested
@DisplayName("Remark-aware fixture workflow") @DisplayName("Remark-aware fixture workflow")
@Tag("integration")
@Tag("cli")
@Tag("stemmer")
@Tag("compile")
@Tag("construction")
final class RemarkAwareFixtureWorkflow { final class RemarkAwareFixtureWorkflow {
/** /**
@@ -189,9 +197,10 @@ final class CompileIntegrationTest {
* create nested output directories, preserve expected lookup behavior, and * create nested output directories, preserve expected lookup behavior, and
* store canonical stems when {@code --store-original} is enabled. * store canonical stems when {@code --store-original} is enabled.
* *
* @throws IOException if reading or writing fails * @throws IOException if reading or writing fails
*/ */
@Test @Test
@Tag("slow")
@DisplayName("CLI should compile the remark-aware fixture and preserve expected lookups") @DisplayName("CLI should compile the remark-aware fixture and preserve expected lookups")
void shouldCompileRemarkAwareFixtureAndPreserveExpectedLookups() throws IOException { void shouldCompileRemarkAwareFixtureAndPreserveExpectedLookups() throws IOException {
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE, final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
@@ -234,9 +243,10 @@ final class CompileIntegrationTest {
* Verifies that the CLI rejects an already existing output path unless * Verifies that the CLI rejects an already existing output path unless
* overwrite is explicitly enabled. * overwrite is explicitly enabled.
* *
* @throws IOException if reading or writing fails * @throws IOException if reading or writing fails
*/ */
@Test @Test
@Tag("slow")
@DisplayName("CLI should require overwrite before replacing an existing output artifact") @DisplayName("CLI should require overwrite before replacing an existing output artifact")
void shouldRequireOverwriteForExistingOutput() throws IOException { void shouldRequireOverwriteForExistingOutput() throws IOException {
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE, final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
@@ -301,6 +311,12 @@ final class CompileIntegrationTest {
@Nested @Nested
@DisplayName("Bundled project dictionary workflows") @DisplayName("Bundled project dictionary workflows")
@Tag("slow")
@Tag("integration")
@Tag("cli")
@Tag("stemmer")
@Tag("compile")
@Tag("construction")
final class BundledProjectDictionaryWorkflows { final class BundledProjectDictionaryWorkflows {
/** /**
@@ -317,11 +333,12 @@ final class CompileIntegrationTest {
* </p> * </p>
* *
* @param scenario scenario identifier * @param scenario scenario identifier
* @param resourcePath bundled dictionary resource path * @param resourcePath bundled dictionary resource path
* @throws IOException if reading or writing fails * @throws IOException if reading or writing fails
*/ */
@ParameterizedTest(name = "[{index}] {0}") @ParameterizedTest(name = "[{index}] {0}")
@MethodSource("org.egothor.stemmer.CompileIntegrationTest#bundledDictionaryCases") @MethodSource("org.egothor.stemmer.CompileIntegrationTest#bundledDictionaryCases")
@Tag("slow")
@DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics") @DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics")
void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario, void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario,
final String resourcePath) throws IOException { final String resourcePath) throws IOException {

View File

@@ -66,7 +66,11 @@ import org.junit.jupiter.api.io.TempDir;
* {@link System#exit(int)}. * {@link System#exit(int)}.
* </p> * </p>
*/ */
@Tag("unit") @Tag("integration")
@Tag("cli")
@Tag("compile")
@Tag("stemmer")
@Tag("construction")
@DisplayName("Compile") @DisplayName("Compile")
class CompileTest { class CompileTest {
@@ -175,6 +179,11 @@ class CompileTest {
@Nested @Nested
@DisplayName("argument validation") @DisplayName("argument validation")
@Tag("integration")
@Tag("cli")
@Tag("compile")
@Tag("stemmer")
@Tag("validation")
class ArgumentValidationTest { class ArgumentValidationTest {
@Test @Test

View File

@@ -70,10 +70,11 @@ import org.junit.jupiter.params.provider.MethodSource;
* <li>compressed artifact reproducibility within the active format version</li> * <li>compressed artifact reproducibility within the active format version</li>
* </ul> * </ul>
*/ */
@Tag("unit") @Tag("compat")
@Tag("regression") @Tag("regression")
@Tag("determinism") @Tag("determinism")
@Tag("serialization") @Tag("serialization")
@Tag("trie")
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
final class CompiledTrieArtifactRegressionTest { final class CompiledTrieArtifactRegressionTest {

View File

@@ -41,7 +41,9 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link DiacriticStripper}. * Unit tests for {@link DiacriticStripper}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("diacritics") @Tag("diacritic")
@Tag("stemmer")
@Tag("normalization")
@DisplayName("DiacriticStripper") @DisplayName("DiacriticStripper")
class DiacriticStripperTest { class DiacriticStripperTest {

View File

@@ -59,7 +59,7 @@ import org.junit.jupiter.api.Test;
*/ */
@DisplayName("FrequencyTrieBuilders") @DisplayName("FrequencyTrieBuilders")
@Tag("unit") @Tag("unit")
@Tag("builder") @Tag("construction")
@Tag("frequency-trie") @Tag("frequency-trie")
class FrequencyTrieBuildersTest { class FrequencyTrieBuildersTest {

View File

@@ -47,7 +47,7 @@ import java.util.List;
import net.jqwik.api.ForAll; import net.jqwik.api.ForAll;
import net.jqwik.api.Label; import net.jqwik.api.Label;
import net.jqwik.api.Property; import net.jqwik.api.Property;
import net.jqwik.api.Tag; import org.junit.jupiter.api.Tag;
/** /**
* Property-based tests for the compiled trie abstraction. * Property-based tests for the compiled trie abstraction.
@@ -59,9 +59,9 @@ import net.jqwik.api.Tag;
* core algorithm without overfitting to particular fixture data. * core algorithm without overfitting to particular fixture data.
*/ */
@Label("FrequencyTrie properties") @Label("FrequencyTrie properties")
@Tag("unit")
@Tag("property") @Tag("property")
@Tag("trie") @Tag("trie")
@Tag("frequency-trie")
class FrequencyTrieProperties extends PropertyBasedTestSupport { class FrequencyTrieProperties extends PropertyBasedTestSupport {
/** /**

View File

@@ -33,6 +33,7 @@ package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertSame;
@@ -44,6 +45,7 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream; import java.io.DataInputStream;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.DisplayName;
@@ -62,6 +64,7 @@ import org.junit.jupiter.api.Test;
@Tag("unit") @Tag("unit")
@Tag("trie") @Tag("trie")
@Tag("frequency-trie") @Tag("frequency-trie")
@Tag("lookup")
@DisplayName("FrequencyTrie") @DisplayName("FrequencyTrie")
class FrequencyTrieTest { class FrequencyTrieTest {
@@ -379,6 +382,167 @@ class FrequencyTrieTest {
assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1))); assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1)));
} }
/**
* Verifies that {@link FrequencyTrie#getEntries(String)} short-circuits to a one-item immutable list.
*/
@Test
@DisplayName("getEntries returns a one-item list for single stored values")
void getEntriesReturnsSingleItemListForSingleStoredValue() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("gamma", "only");
final FrequencyTrie<String> trie = builder.build();
final List<ValueCount<String>> entries = trie.getEntries("gamma");
assertAll(() -> assertEquals(List.of(new ValueCount<String>("only", 1)), entries),
() -> assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1))));
}
/**
* Verifies that visitor lookup returns the same deterministic order and counts
* as the allocating APIs.
*/
@Test
@DisplayName("Visitor lookup matches getAll order and getEntries counts")
void visitorLookupMatchesGetAllOrderAndGetEntriesCounts() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun", 3);
builder.put("house", "verb", 2);
builder.put("house", "adjective", 1);
final FrequencyTrie<String> trie = builder.build();
final List<String> values = new ArrayList<>();
final List<Integer> counts = new ArrayList<>();
final List<Integer> ranks = new ArrayList<>();
final int visited = trie.getAllNormalized("house", (value, count, rank) -> {
values.add(value);
counts.add(count);
ranks.add(rank);
return true;
}, 10);
assertAll(() -> assertEquals(3, visited),
() -> assertEquals(List.of("noun", "verb", "adjective"), values),
() -> assertEquals(List.of(3, 2, 1), counts),
() -> assertEquals(List.of(0, 1, 2), ranks));
}
/**
* Verifies visitor maximum result and early-stop behavior.
*/
@Test
@DisplayName("Visitor lookup honors maxResults and sink early stop")
void visitorLookupHonorsMaxResultsAndSinkEarlyStop() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun", 3);
builder.put("house", "verb", 2);
builder.put("house", "adjective", 1);
final FrequencyTrie<String> trie = builder.build();
final List<String> limited = new ArrayList<>();
final List<String> stopped = new ArrayList<>();
final int limitedCount = trie.getAllNormalized("house", (value, count, rank) -> {
limited.add(value);
return true;
}, 2);
final int stoppedCount = trie.getAllNormalized("house", (value, count, rank) -> {
stopped.add(value);
return false;
}, 10);
assertAll(() -> assertEquals(2, limitedCount),
() -> assertEquals(List.of("noun", "verb"), limited),
() -> assertEquals(1, stoppedCount),
() -> assertEquals(List.of("noun"), stopped));
}
/**
* Verifies visitor zero, negative, missing, and first-result behavior.
*/
@Test
@DisplayName("Visitor lookup handles zero, negative, missing, and first-result cases")
void visitorLookupHandlesBoundaryCases() {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("house", "noun");
final FrequencyTrie<String> trie = builder.build();
final int[] calls = new int[1];
assertAll(() -> assertEquals(0, trie.getAllNormalized("house", (value, count, rank) -> {
calls[0]++;
return true;
}, 0)),
() -> assertEquals(0, calls[0]),
() -> assertThrows(IllegalArgumentException.class,
() -> trie.getAllNormalized("house", (value, count, rank) -> true, -1)),
() -> assertEquals(0, trie.getAllNormalized("missing", (value, count, rank) -> true, 10)),
() -> assertFalse(trie.getFirstNormalized("missing", (value, count, rank) -> true)),
() -> assertTrue(trie.getFirstNormalized("house", (value, count, rank) -> {
assertEquals("noun", value);
assertEquals(1, count);
assertEquals(0, rank);
return true;
})));
}
/**
* Verifies visitor API argument validation.
*/
@Test
@DisplayName("Visitor lookup rejects null and invalid range arguments")
void visitorLookupRejectsNullAndInvalidRangeArguments() {
final FrequencyTrie<String> trie = rankedBuilder().build();
final char[] key = "house".toCharArray();
final FrequencyTrie.EntrySink<String> sink = (value, count, rank) -> true;
assertAll(() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized((char[]) null, 0, 0, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized(key, 0, key.length, null, 1)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> trie.getAllNormalized(key, -1, key.length, sink, 1)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> trie.getAllNormalized(key, 1, key.length, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAllNormalized("house", null, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAll((CharSequence) null, sink, 1)),
() -> assertThrows(NullPointerException.class,
() -> trie.getAll("house", null, 1)),
() -> assertThrows(IllegalArgumentException.class,
() -> trie.getAll("house", sink, -1)));
}
/**
* Verifies normalized char-array slices and metadata-aware CharSequence visitor
* lookup.
*/
@Test
@DisplayName("Visitor lookup supports normalized char slices and metadata-aware CharSequence keys")
void visitorLookupSupportsCharSlicesAndMetadataAwareKeys() {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
builder.put("house", "noun");
final FrequencyTrie<String> trie = builder.build();
final char[] padded = "__house__".toCharArray();
assertAll(() -> assertEquals(1,
trie.getAllNormalized(padded, 2, 5, (value, count, rank) -> {
assertEquals("noun", value);
return true;
}, 10)),
() -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true),
"Normalized lookup must bypass metadata lowercasing."),
() -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> {
assertEquals("noun", value);
return true;
})));
}
/** /**
* Verifies that equal frequencies prefer the shorter string representation. * Verifies that equal frequencies prefer the shorter string representation.
*/ */
@@ -709,6 +873,7 @@ class FrequencyTrieTest {
.readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC); .readFrom(new ByteArrayInputStream(outputStream.toByteArray()), String[]::new, STRING_CODEC);
assertAll(() -> assertEquals(original.size(), restored.size()), assertAll(() -> assertEquals(original.size(), restored.size()),
() -> assertEquals(original.getFingerprint(), restored.getFingerprint()),
() -> assertEquals(original.get(""), restored.get("")), () -> assertEquals(original.get(""), restored.get("")),
() -> assertArrayEquals(original.getAll(""), restored.getAll("")), () -> assertArrayEquals(original.getAll(""), restored.getAll("")),
() -> assertEquals(original.get("run"), restored.get("run")), () -> assertEquals(original.get("run"), restored.get("run")),
@@ -728,6 +893,82 @@ class FrequencyTrieTest {
() -> assertEquals(List.of(), restored.getEntries("missing"))); () -> assertEquals(List.of(), restored.getEntries("missing")));
} }
/**
* Verifies fingerprint stability and sensitivity to metadata and trie content.
*/
@Test
@DisplayName("Fingerprint reflects metadata and compiled trie content")
void fingerprintReflectsMetadataAndCompiledTrieContent() {
final FrequencyTrie.Builder<String> baseBuilderA = rankedBuilder();
baseBuilderA.put("run", "verb", 3);
baseBuilderA.put("run", "noun", 1);
baseBuilderA.put("runner", "noun", 2);
final FrequencyTrie<String> trieA = baseBuilderA.build();
final FrequencyTrie.Builder<String> baseBuilderB = rankedBuilder();
baseBuilderB.put("run", "verb", 3);
baseBuilderB.put("run", "noun", 1);
baseBuilderB.put("runner", "noun", 2);
final FrequencyTrie<String> trieB = baseBuilderB.build();
final FrequencyTrie.Builder<String> reorderedBuilder = rankedBuilder();
reorderedBuilder.put("runner", "noun", 2);
reorderedBuilder.put("run", "noun", 1);
reorderedBuilder.put("run", "verb", 3);
final FrequencyTrie<String> reorderedTrie = reorderedBuilder.build();
final FrequencyTrie.Builder<String> differentContentBuilder = rankedBuilder();
differentContentBuilder.put("run", "verb", 3);
differentContentBuilder.put("run", "noun", 2);
differentContentBuilder.put("runner", "noun", 2);
final FrequencyTrie<String> differentContentTrie = differentContentBuilder.build();
final FrequencyTrie.Builder<String> differentMetadataBuilder = new FrequencyTrie.Builder<>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
WordTraversalDirection.FORWARD, CaseProcessingMode.AS_IS);
differentMetadataBuilder.put("run", "verb", 3);
differentMetadataBuilder.put("run", "noun", 1);
differentMetadataBuilder.put("runner", "noun", 2);
final FrequencyTrie<String> differentMetadataTrie = differentMetadataBuilder.build();
final String fingerprintA = trieA.getFingerprint();
final String fingerprintB = trieB.getFingerprint();
final String reorderedFingerprint = reorderedTrie.getFingerprint();
final String differentContentFingerprint = differentContentTrie.getFingerprint();
final String differentMetadataFingerprint = differentMetadataTrie.getFingerprint();
final byte[] fingerprintBytes = trieA.copyFingerprintBytes();
final byte[] secondFingerprintBytes = trieA.copyFingerprintBytes();
fingerprintBytes[0] = (byte) (fingerprintBytes[0] ^ 0x7F);
assertAll(() -> assertEquals(fingerprintA, fingerprintB),
() -> assertEquals(fingerprintA, reorderedFingerprint),
() -> assertEquals(fingerprintA, trieA.getFingerprint()),
() -> assertFalse(fingerprintA.isBlank()),
() -> assertLowercaseSha256Hex(fingerprintA),
() -> assertEquals(fingerprintA, toLowerHex(secondFingerprintBytes)),
() -> assertArrayEquals(secondFingerprintBytes, trieA.copyFingerprintBytes()),
() -> assertFalse(fingerprintA.equals(differentContentFingerprint)),
() -> assertFalse(fingerprintA.equals(differentMetadataFingerprint)));
}
private static void assertLowercaseSha256Hex(final String fingerprint) {
assertEquals(64, fingerprint.length());
for (int index = 0; index < fingerprint.length(); index++) {
final char character = fingerprint.charAt(index);
final boolean digit = character >= '0' && character <= '9';
final boolean lowercaseHex = character >= 'a' && character <= 'f';
assertTrue(digit || lowercaseHex, "Invalid fingerprint character at index " + index + '.');
}
}
private static String toLowerHex(final byte[] bytes) {
final StringBuilder builder = new StringBuilder(bytes.length * 2);
for (byte item : bytes) {
builder.append(Character.forDigit((item >>> 4) & 0x0F, 16));
builder.append(Character.forDigit(item & 0x0F, 16));
}
return builder.toString();
}
/** /**
* Verifies that persistence methods reject {@code null} arguments. * Verifies that persistence methods reject {@code null} arguments.
* *
@@ -755,6 +996,115 @@ class FrequencyTrieTest {
.readFrom(new ByteArrayInputStream(serializedEmptyTrie), String[]::new, null))); .readFrom(new ByteArrayInputStream(serializedEmptyTrie), String[]::new, null)));
} }
/**
* Verifies that reading a compiled trie with a negative max-expanded override
* smaller than -1 is rejected.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom rejects invalid maxExpandedIndex override")
void readFromRejectsInvalidMaxExpandedIndexOverride() {
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC, -2));
assertEquals("maxExpandedIndex must be >= -1.", exception.getMessage());
}
/**
* Verifies that the max-expanded override controls dense lookup materialization
* while preserving lookup semantics.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom respects dense lookup max-expanded index override")
void readFromRespectsDenseLookupMaxExpandedIndexOverride() throws IOException {
final FrequencyTrie.Builder<String> builder = rankedBuilder();
builder.put("a", "a");
builder.put("b", "b");
builder.put("c", "c");
builder.put("d", "d");
final FrequencyTrie<String> original = builder.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
original.writeTo(outputStream, STRING_CODEC);
final byte[] serializedTrie = outputStream.toByteArray();
final FrequencyTrie<String> defaultDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
STRING_CODEC);
final FrequencyTrie<String> defaultDenseByNegative = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie),
String[]::new, STRING_CODEC, -1);
final FrequencyTrie<String> disabledDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
STRING_CODEC, 0);
assertAll(
() -> assertTrue(defaultDense.root().hasDenseLookup(),
"Default read should enable dense lookup for compact first-level edges."),
() -> assertTrue(defaultDenseByNegative.root().hasDenseLookup(),
"Negative override should use the default dense lookup span."),
() -> assertFalse(disabledDense.root().hasDenseLookup(),
"Zero override should disable dense lookup tables."),
() -> assertEquals(original.get("a"), disabledDense.get("a")),
() -> assertEquals(original.get("b"), disabledDense.get("b")),
() -> assertEquals(original.get("c"), disabledDense.get("c")),
() -> assertEquals(original.get("d"), disabledDense.get("d")),
() -> assertEquals(original.get("z"), disabledDense.get("z")));
}
/**
* Verifies that cyclic serialized node references are rejected as invalid
* serialization.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom rejects cyclic serialized node references")
void readFromRejectsCyclicSerializedNodeReferences() {
final byte[] bytes = createSerializedStream(0x45475452, 1, 2, 0, new NodeWriter[] {
dataOutput -> {
dataOutput.writeInt(1);
dataOutput.writeChar('b');
dataOutput.writeInt(1);
dataOutput.writeInt(0);
},
dataOutput -> {
dataOutput.writeInt(1);
dataOutput.writeChar('a');
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertTrue(exception.getMessage().contains("cyclic reference detected"));
}
/**
* Verifies that child node references outside the valid serialized range are
* rejected.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom rejects invalid child node identifiers")
void readFromRejectsInvalidChildNodeId() {
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(1);
dataOutput.writeChar('a');
dataOutput.writeInt(3);
dataOutput.writeInt(0);
} });
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertTrue(exception.getMessage().contains("Invalid child node id"));
}
/** /**
* Verifies that deserialization rejects an invalid stream magic header. * Verifies that deserialization rejects an invalid stream magic header.
*/ */
@@ -785,6 +1135,27 @@ class FrequencyTrieTest {
assertTrue(exception.getMessage().contains("Unsupported trie stream version")); assertTrue(exception.getMessage().contains("Unsupported trie stream version"));
} }
/**
* Verifies that the latest stream version validates textual metadata blocks.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom rejects invalid textual metadata block")
void readFromRejectsInvalidTextualMetadataBlock() {
final int version = FrequencyTrie.currentFormatVersion();
final byte[] bytes = createSerializedStream(0x45475452, version, 1, 0, dataOutput -> {
dataOutput.writeUTF("not valid metadata");
}, new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertTrue(exception.getMessage().contains("Invalid metadata block"));
}
/** /**
* Verifies that deserialization rejects a negative node count. * Verifies that deserialization rejects a negative node count.
*/ */
@@ -862,6 +1233,129 @@ class FrequencyTrieTest {
assertTrue(exception.getMessage().contains("Non-positive stored count")); assertTrue(exception.getMessage().contains("Non-positive stored count"));
} }
/**
* Verifies that legacy version 1 metadata uses compatibility defaults.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom supports legacy version 1 metadata")
void readFromSupportsLegacyVersionOneMetadata() throws IOException {
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
assertEquals(TrieMetadata.legacy(1, WordTraversalDirection.BACKWARD), trie.metadata());
}
/**
* Verifies that legacy version 2 metadata stores traversal direction and uses
* compatibility defaults for other values.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom supports legacy version 2 metadata")
void readFromSupportsLegacyVersionTwoMetadata() throws IOException {
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
dataOutput -> dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal()), new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
assertEquals(TrieMetadata.legacy(2, WordTraversalDirection.FORWARD), trie.metadata());
}
/**
* Verifies that version 3 metadata includes reduction and diacritic
* processing settings.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom parses version 3 metadata")
void readFromParsesVersionThreeMetadata() throws IOException {
final ReductionSettings reductionSettings = new ReductionSettings(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS, 81, 4);
final byte[] bytes = createSerializedStream(0x45475452, 3, 1, 0,
dataOutput -> {
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
dataOutput.writeInt(DiacriticProcessingMode.REMOVE.ordinal());
},
new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
final TrieMetadata metadata = trie.metadata();
assertAll(() -> assertEquals(3, metadata.formatVersion()),
() -> assertEquals(WordTraversalDirection.BACKWARD, metadata.traversalDirection()),
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
() -> assertEquals(DiacriticProcessingMode.REMOVE, metadata.diacriticProcessingMode()),
() -> assertEquals(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, metadata.caseProcessingMode()));
}
/**
* Verifies that version 4 metadata additionally stores case-processing mode.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom parses version 4 case processing metadata")
void readFromParsesVersionFourCaseMetadata() throws IOException {
final ReductionSettings reductionSettings = new ReductionSettings(
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, 75, 3);
final byte[] bytes = createSerializedStream(0x45475452, 4, 1, 0,
dataOutput -> {
dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal());
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal());
dataOutput.writeInt(CaseProcessingMode.AS_IS.ordinal());
},
new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
final TrieMetadata metadata = trie.metadata();
assertAll(() -> assertEquals(4, metadata.formatVersion()),
() -> assertEquals(WordTraversalDirection.FORWARD, metadata.traversalDirection()),
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
() -> assertEquals(DiacriticProcessingMode.AS_IS, metadata.diacriticProcessingMode()),
() -> assertEquals(CaseProcessingMode.AS_IS, metadata.caseProcessingMode()));
}
/**
* Verifies that invalid legacy metadata ordinals are rejected by validation.
*/
@Test
@Tag("persistence")
@DisplayName("readFrom rejects invalid metadata ordinal in legacy stream")
void readFromRejectsInvalidLegacyMetadataOrdinal() {
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
dataOutput -> dataOutput.writeInt(999), new NodeWriter[] { dataOutput -> {
dataOutput.writeInt(0);
dataOutput.writeInt(0);
} });
final IOException exception = assertThrows(IOException.class,
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
assertTrue(exception.getMessage().contains("Invalid traversal direction ordinal"));
}
/** /**
* Writes one node body into a synthetic serialized trie stream. * Writes one node body into a synthetic serialized trie stream.
*/ */
@@ -889,6 +1383,24 @@ class FrequencyTrieTest {
*/ */
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount, private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
final int rootNodeId, final NodeWriter[] nodes) { final int rootNodeId, final NodeWriter[] nodes) {
return createSerializedStream(magic, version, nodeCount, rootNodeId, dataOutput -> {
// legacy and text-based versions write their metadata differently.
}, nodes);
}
/**
* Writes a synthetic serialized trie stream with a metadata writer hook.
*
* @param magic stream magic
* @param version stream version
* @param nodeCount declared node count
* @param rootNodeId declared root node identifier
* @param metadata version-specific metadata writer
* @param nodes node body writers
* @return serialized bytes
*/
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
final int rootNodeId, final MetadataWriter metadata, final NodeWriter[] nodes) {
try { try {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
@@ -897,6 +1409,7 @@ class FrequencyTrieTest {
dataOutputStream.writeInt(version); dataOutputStream.writeInt(version);
dataOutputStream.writeInt(nodeCount); dataOutputStream.writeInt(nodeCount);
dataOutputStream.writeInt(rootNodeId); dataOutputStream.writeInt(rootNodeId);
metadata.write(dataOutputStream);
for (NodeWriter node : nodes) { for (NodeWriter node : nodes) {
node.write(dataOutputStream); node.write(dataOutputStream);
@@ -908,4 +1421,19 @@ class FrequencyTrieTest {
throw new IllegalStateException("Unexpected I/O while building synthetic trie stream.", exception); throw new IllegalStateException("Unexpected I/O while building synthetic trie stream.", exception);
} }
} }
/**
* Writes one synthetic metadata block.
*/
@FunctionalInterface
private interface MetadataWriter {
/**
* Writes metadata bytes for one stream version.
*
* @param dataOutput output stream
* @throws IOException if writing fails
*/
void write(DataOutputStream dataOutput) throws IOException;
}
} }

View File

@@ -65,10 +65,10 @@ import org.junit.jupiter.api.io.TempDir;
* stems declared by the source dictionary. * stems declared by the source dictionary.
*/ */
@DisplayName("Deterministic fuzz-style trie and stemmer compilation") @DisplayName("Deterministic fuzz-style trie and stemmer compilation")
@Tag("unit")
@Tag("fuzz") @Tag("fuzz")
@Tag("trie") @Tag("trie")
@Tag("stemming") @Tag("stemmer")
@Tag("determinism")
class FuzzStemmerAndTrieCompilationTest { class FuzzStemmerAndTrieCompilationTest {
/** /**

View File

@@ -36,7 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import net.jqwik.api.ForAll; import net.jqwik.api.ForAll;
import net.jqwik.api.Label; import net.jqwik.api.Label;
import net.jqwik.api.Property; import net.jqwik.api.Property;
import net.jqwik.api.Tag; import org.junit.jupiter.api.Tag;
/** /**
* Property-based tests for {@link PatchCommandEncoder}. * Property-based tests for {@link PatchCommandEncoder}.
@@ -47,9 +47,9 @@ import net.jqwik.api.Tag;
* reconstruct the exact requested target. * reconstruct the exact requested target.
*/ */
@Label("PatchCommandEncoder properties") @Label("PatchCommandEncoder properties")
@Tag("unit")
@Tag("property") @Tag("property")
@Tag("patch") @Tag("patch")
@Tag("stemmer")
class PatchCommandEncoderProperties extends PropertyBasedTestSupport { class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
/** /**

View File

@@ -31,6 +31,7 @@
package org.egothor.stemmer; package org.egothor.stemmer;
import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
@@ -68,6 +69,8 @@ import org.junit.jupiter.params.provider.MethodSource;
@Tag("unit") @Tag("unit")
@Tag("stemmer") @Tag("stemmer")
@Tag("patch") @Tag("patch")
@Tag("encoding")
@Tag("apply")
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class PatchCommandEncoderTest { class PatchCommandEncoderTest {
@@ -147,6 +150,63 @@ class PatchCommandEncoderTest {
Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher")); Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher"));
} }
/**
* Provides explicit forward-direction single-instruction patch application cases.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardSingleInstructionApplyCases() {
return Stream.of(
// 1
Arguments.of(1, "abcd", "Db", "cd"),
// 2
Arguments.of(2, "abc", "Ia", "aabc"),
// 3
Arguments.of(3, "abc", "Ra", "abc"),
// 4
Arguments.of(4, "abc", "-a", "abc"),
// 5
Arguments.of(5, "abc", PatchCommandEncoder.NOOP_PATCH, "abc"));
}
/**
* Provides forward-direction applyTo cases that exercise preserve-only and
* non-preserve branches.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardApplyToCases() {
return Stream.of(
// 1
Arguments.of(1, "book", "-aRa", "baok"),
// 2
Arguments.of(2, "abc", "-dRa", "abc"),
// 3
Arguments.of(3, "abc", "DdRa", "abc"),
// 4
Arguments.of(4, "abc", "-dIa", "abc"),
// 5
Arguments.of(5, "abc", "Na-a", "abc"));
}
/**
* Provides empty-source forward applyTo cases that cover empty-source
* instruction handling.
*
* @return test arguments
*/
private static Stream<Arguments> provideForwardEmptySourceApplyCases() {
return Stream.of(
// 1
Arguments.of(1, "IaIb", "ab"),
// 2
Arguments.of(2, "-aRa", ""),
// 3
Arguments.of(3, "IaRa", ""),
// 4
Arguments.of(4, "Na-a", ""));
}
/** /**
* Provides malformed or index-invalid patch inputs that must preserve the * Provides malformed or index-invalid patch inputs that must preserve the
* original source according to the implementation contract. * original source according to the implementation contract.
@@ -236,12 +296,31 @@ class PatchCommandEncoderTest {
return new StringBuilder(text).reverse().toString(); return new StringBuilder(text).reverse().toString();
} }
/**
* Applies a patch into a right-sized output buffer and returns the produced
* string.
*
* @param source source text
* @param patch patch command
* @param traversalDirection traversal direction
* @return transformed text
*/
private static String applyToString(final String source, final String patch,
final WordTraversalDirection traversalDirection) {
final char[] output = new char[Math.max(source.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo(source, patch, traversalDirection, output, 0, output.length);
return new String(output, 0, produced);
}
/** /**
* Tests constructor validation and basic instantiation behavior. * Tests constructor validation and basic instantiation behavior.
*/ */
@Nested @Nested
@DisplayName("construction") @DisplayName("construction")
@Tag("constructor") @Tag("construction")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ConstructionTests { class ConstructionTests {
/** /**
@@ -326,7 +405,10 @@ class PatchCommandEncoderTest {
*/ */
@Nested @Nested
@DisplayName("encode(String, String)") @DisplayName("encode(String, String)")
@Tag("encode") @Tag("encoding")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class EncodeTests { class EncodeTests {
/** /**
@@ -461,6 +543,9 @@ class PatchCommandEncoderTest {
@Nested @Nested
@DisplayName("apply(String, String)") @DisplayName("apply(String, String)")
@Tag("apply") @Tag("apply")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ApplyTests { class ApplyTests {
/** /**
@@ -535,6 +620,101 @@ class PatchCommandEncoderTest {
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD)); assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
} }
/**
* Verifies explicit single-instruction forward patch application
* semantics.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: forward single instruction apply({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardSingleInstructionApplyCases")
@DisplayName("applies forward single instruction patches correctly")
void shouldApplyForwardSingleInstructionsExplicitly(int caseId, String source, String patch, String expected) {
assertEquals(expected, PatchCommandEncoder.apply(source, patch, WordTraversalDirection.FORWARD),
() -> "Case " + caseId + " failed for source='" + source + "', patch='" + patch + "'.");
}
/**
* Verifies forward single-instruction malformed commands fail fast.
*/
@Test
@DisplayName("throws for unsupported forward opcode and NOOP argument")
void shouldThrowForUnsupportedForwardOpcodeAndNoopArgument() {
assertAll(() -> assertEquals("Unsupported patch opcode: X",
assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("abc", "Xa", WordTraversalDirection.FORWARD))
.getMessage()),
() -> assertEquals("Unsupported NOOP patch argument: `",
assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("abc", "N`", WordTraversalDirection.FORWARD))
.getMessage()));
}
/**
* Verifies explicit forward-applyTo cases that exercise preserve-only and
* non-preserve branches.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyToForward({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardApplyToCases")
@DisplayName("applyTo handles forward preserve-only and mutation branches")
void shouldApplyToForwardPreserveAndMutationBranches(int caseId, String source, String patch, String expected) {
final char[] output = new char[Math.max(source.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.FORWARD, output, 0,
output.length);
assertAll(
() -> assertEquals(expected.length(), produced,
() -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 0, produced),
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
}
/**
* Verifies empty-source forward applyTo behavior for insert-only and malformed
* instructions.
*
* @param caseId numeric case identifier
* @param patch encoded patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyToForward(\"\", {1}) -> \"{2}\"")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardEmptySourceApplyCases")
@DisplayName("applies forward empty-source patches correctly")
void shouldApplyToForwardEmptySourceCases(int caseId, String patch, String expected) {
final char[] output = new char[Math.max(expected.length() + 16, 16)];
final int produced = PatchCommandEncoder.applyTo("", patch, WordTraversalDirection.FORWARD, output, 0,
output.length);
assertAll(
() -> assertEquals(expected.length(), produced,
() -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 0, produced),
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
}
/**
* Verifies malformed empty-source forward patches fail fast and preserve
* empty-source semantics.
*/
@Test
@DisplayName("throws for unsupported NOOP argument on empty-source forward patch")
void shouldThrowForUnsupportedNoopArgumentOnForwardEmptySource() {
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.apply("", "N`Ra", WordTraversalDirection.FORWARD));
assertEquals("Unsupported NOOP patch argument: `", exception.getMessage());
}
/** /**
* Verifies explicit patch application cases. * Verifies explicit patch application cases.
* *
@@ -590,6 +770,181 @@ class PatchCommandEncoderTest {
assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId
+ " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'."); + " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'.");
} }
/**
* Verifies buffer application against string-returning application.
*
* @param caseId numeric case identifier
* @param source source word
* @param patch patch command
* @param expected expected transformed word
*/
@ParameterizedTest(name = "[{index}] case {0}: applyTo({1}, {2}) -> {3}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideApplyCases")
@DisplayName("applyTo matches apply for explicit backward patch commands")
void shouldApplyToBufferLikeApplyForBackwardCommands(int caseId, String source, String patch, String expected) {
final char[] output = "___..............".toCharArray();
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.BACKWARD, output, 3,
output.length - 3);
assertAll(() -> assertEquals(expected.length(), produced, () -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(expected, new String(output, 3, produced)));
}
/**
* Verifies char-array source slices.
*/
@Test
@DisplayName("applyTo supports char-array source slices")
void shouldApplyToCharArraySourceSlice() {
final char[] source = "__teacher__".toCharArray();
final char[] output = new char[16];
final int produced = PatchCommandEncoder.applyTo(source, 2, 7, "Db", WordTraversalDirection.BACKWARD,
output, 1, output.length - 1);
assertAll(() -> assertEquals(5, produced), () -> assertEquals("teach", new String(output, 1, produced)));
}
/**
* Verifies null and range validation for buffer application.
*/
@Test
@DisplayName("applyTo rejects null and invalid range arguments")
void shouldRejectNullAndInvalidRangeArguments() {
final char[] source = "teacher".toCharArray();
final char[] output = new char[16];
assertAll(() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo((CharSequence) null, "Db", WordTraversalDirection.BACKWARD,
output, 0, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", null, output, 0, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD, null,
0, output.length)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD,
output, -1, output.length)),
() -> assertThrows(NullPointerException.class,
() -> PatchCommandEncoder.applyTo((char[]) null, 0, 7, "Db",
WordTraversalDirection.BACKWARD, output, 0, output.length)),
() -> assertThrows(IndexOutOfBoundsException.class,
() -> PatchCommandEncoder.applyTo(source, 1, source.length, "Db",
WordTraversalDirection.BACKWARD, output, 0, output.length)));
}
/**
* Verifies null, empty, and canonical NOOP patch preservation.
*/
@Test
@DisplayName("applyTo preserves source for null, empty, and canonical NOOP patches")
void shouldApplyToPreserveSourceForEmptyCompatibilityPatches() {
final char[] nullPatchOutput = new char[8];
final char[] emptyPatchOutput = new char[8];
final char[] noopOutput = new char[8];
final int nullPatchLength = PatchCommandEncoder.applyTo("teacher", null, WordTraversalDirection.BACKWARD,
nullPatchOutput, 0, nullPatchOutput.length);
final int emptyPatchLength = PatchCommandEncoder.applyTo("teacher", "", WordTraversalDirection.BACKWARD,
emptyPatchOutput, 0, emptyPatchOutput.length);
final int noopLength = PatchCommandEncoder.applyTo("teacher", PatchCommandEncoder.NOOP_PATCH,
WordTraversalDirection.BACKWARD, noopOutput, 0, noopOutput.length);
assertAll(() -> assertEquals(7, nullPatchLength),
() -> assertEquals("teacher", new String(nullPatchOutput, 0, nullPatchLength)),
() -> assertEquals(7, emptyPatchLength),
() -> assertEquals("teacher", new String(emptyPatchOutput, 0, emptyPatchLength)),
() -> assertEquals(7, noopLength),
() -> assertEquals("teacher", new String(noopOutput, 0, noopLength)));
}
/**
* Verifies insufficient capacity behavior.
*/
@Test
@DisplayName("applyTo reports insufficient capacity without writing output")
void shouldReportInsufficientCapacityWithoutWritingOutput() {
final char[] output = "xxxx".toCharArray();
final int produced = PatchCommandEncoder.applyTo("abc", "Ic", WordTraversalDirection.BACKWARD, output, 0,
output.length - 1);
assertAll(() -> assertEquals(PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY, produced),
() -> assertArrayEquals("xxxx".toCharArray(), output));
}
/**
* Verifies exception parity with string-returning application.
*/
@Test
@DisplayName("applyTo throws for unsupported opcode and NOOP argument")
void shouldApplyToThrowForUnsupportedOpcodeAndNoopArgument() {
final char[] output = new char[8];
assertAll(() -> {
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo("abc", "Xa", WordTraversalDirection.BACKWARD, output, 0,
output.length));
assertEquals("Unsupported patch opcode: X", exception.getMessage());
}, () -> {
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo("abc", "Nb", WordTraversalDirection.BACKWARD, output, 0,
output.length));
assertEquals("Unsupported NOOP patch argument: b", exception.getMessage());
});
}
/**
* Verifies malformed compatibility behavior for buffer application.
*
* @param caseId numeric case identifier
* @param source original source
* @param malformedPatch malformed patch
*/
@ParameterizedTest(name = "[{index}] case {0}: malformed applyTo patch {2} preserves {1}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideMalformedPatchCases")
@DisplayName("applyTo preserves source for malformed or index-invalid patch commands")
void shouldApplyToPreserveSourceForMalformedOrIndexInvalidPatchCommands(int caseId, String source,
String malformedPatch) {
final char[] output = new char[Math.max(source.length(), 1)];
final int produced = PatchCommandEncoder.applyTo(source, malformedPatch, WordTraversalDirection.BACKWARD,
output, 0, output.length);
assertAll(() -> assertEquals(source.length(), produced, () -> "Case " + caseId + " produced wrong length."),
() -> assertEquals(source, new String(output, 0, produced)));
}
/**
* Verifies explicit traversal direction for buffer application.
*/
@Test
@DisplayName("applyTo follows explicit forward traversal direction")
void shouldApplyToWithForwardTraversalDirection() {
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
.traversalDirection(WordTraversalDirection.FORWARD)
.build();
final String patch = encoder.encode("cities", "city");
assertEquals(PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD),
applyToString("cities", patch, WordTraversalDirection.FORWARD));
}
/**
* Verifies overlapping source/output slices are rejected.
*/
@Test
@DisplayName("applyTo rejects overlapping char-array source and output ranges")
void shouldRejectOverlappingSourceAndOutputRanges() {
final char[] buffer = "teacher....".toCharArray();
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> PatchCommandEncoder.applyTo(buffer, 0, 7, "Db", WordTraversalDirection.BACKWARD, buffer, 2, 5));
assertEquals("source and output ranges must not overlap.", exception.getMessage());
}
} }
/** /**
@@ -598,6 +953,9 @@ class PatchCommandEncoderTest {
@Nested @Nested
@DisplayName("stemming-oriented scenarios") @DisplayName("stemming-oriented scenarios")
@Tag("regression") @Tag("regression")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class StemmingScenarioTests { class StemmingScenarioTests {
/** /**
@@ -658,7 +1016,10 @@ class PatchCommandEncoderTest {
*/ */
@Nested @Nested
@DisplayName("reversed-word processing") @DisplayName("reversed-word processing")
@Tag("reverse") @Tag("normalization")
@Tag("unit")
@Tag("stemmer")
@Tag("patch")
class ReversedWordProcessingTests { class ReversedWordProcessingTests {
/** /**
@@ -743,6 +1104,7 @@ class PatchCommandEncoderTest {
*/ */
@ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}") @ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}")
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs") @MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
@Tag("normalization")
@DisplayName("preserves correctness under mirrored input orientation") @DisplayName("preserves correctness under mirrored input orientation")
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) { void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build(); PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();

View File

@@ -75,6 +75,8 @@ import org.junit.jupiter.api.io.TempDir;
@DisplayName("StemmerDictionaryParser") @DisplayName("StemmerDictionaryParser")
@Tag("unit") @Tag("unit")
@Tag("parser") @Tag("parser")
@Tag("stemmer")
@Tag("validation")
class StemmerDictionaryParserTest { class StemmerDictionaryParserTest {
/** /**
@@ -97,6 +99,10 @@ class StemmerDictionaryParserTest {
/** /**
* Log handler capturing parser diagnostics for assertions. * Log handler capturing parser diagnostics for assertions.
*/ */
@Tag("unit")
@Tag("parser")
@Tag("stemmer")
@Tag("validation")
private static final class CapturedLogHandler extends Handler { private static final class CapturedLogHandler extends Handler {
/** /**
@@ -157,6 +163,10 @@ class StemmerDictionaryParserTest {
@Nested @Nested
@DisplayName("parse(Reader, String, EntryHandler)") @DisplayName("parse(Reader, String, EntryHandler)")
@Tag("unit")
@Tag("parser")
@Tag("stemmer")
@Tag("validation")
class ReaderParsingTests { class ReaderParsingTests {
@Test @Test
@@ -333,6 +343,10 @@ class StemmerDictionaryParserTest {
@Nested @Nested
@DisplayName("parse(Path, EntryHandler) and parse(String, EntryHandler)") @DisplayName("parse(Path, EntryHandler) and parse(String, EntryHandler)")
@Tag("unit")
@Tag("parser")
@Tag("stemmer")
@Tag("validation")
class FileParsingTests { class FileParsingTests {
@Test @Test
@@ -423,6 +437,10 @@ class StemmerDictionaryParserTest {
@Nested @Nested
@DisplayName("ParseStatistics") @DisplayName("ParseStatistics")
@Tag("unit")
@Tag("parser")
@Tag("stemmer")
@Tag("validation")
class ParseStatisticsTests { class ParseStatisticsTests {
@Test @Test

View File

@@ -54,9 +54,10 @@ import org.junit.jupiter.api.io.TempDir;
/** /**
* Tests for {@link StemmerKnowledgeExperiment}. * Tests for {@link StemmerKnowledgeExperiment}.
*/ */
@Tag("unit")
@Tag("integration") @Tag("integration")
@Tag("stemmer") @Tag("stemmer")
@Tag("trie")
@Tag("reduction")
final class StemmerKnowledgeExperimentTest { final class StemmerKnowledgeExperimentTest {
/** /**

View File

@@ -38,6 +38,8 @@ import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@@ -91,6 +93,8 @@ import org.mockito.MockedStatic;
@Tag("unit") @Tag("unit")
@Tag("io") @Tag("io")
@Tag("persistence") @Tag("persistence")
@Tag("serialization")
@Tag("trie")
@DisplayName("StemmerPatchTrieBinaryIO") @DisplayName("StemmerPatchTrieBinaryIO")
class StemmerPatchTrieBinaryIOTest { class StemmerPatchTrieBinaryIOTest {
@@ -130,6 +134,10 @@ class StemmerPatchTrieBinaryIOTest {
*/ */
@Nested @Nested
@DisplayName("write(...)") @DisplayName("write(...)")
@Tag("unit")
@Tag("io")
@Tag("trie")
@Tag("persistence")
class WriteTests { class WriteTests {
/** /**
@@ -286,6 +294,10 @@ class StemmerPatchTrieBinaryIOTest {
*/ */
@Nested @Nested
@DisplayName("read(...)") @DisplayName("read(...)")
@Tag("unit")
@Tag("io")
@Tag("trie")
@Tag("persistence")
class ReadTests { class ReadTests {
/** /**
@@ -299,9 +311,19 @@ class StemmerPatchTrieBinaryIOTest {
"read(Path) must reject null path."), "read(Path) must reject null path."),
() -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((String) null), () -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((String) null),
"read(String) must reject null file name."), "read(String) must reject null file name."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.read((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"read(Path, int) must reject null path."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.read((String) null,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"read(String, int) must reject null file name."),
() -> assertThrows(NullPointerException.class, () -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null), () -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null),
"read(InputStream) must reject null input stream.")); "read(InputStream) must reject null input stream."),
() -> assertThrows(NullPointerException.class,
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"read(InputStream, int) must reject null input stream."));
} }
/** /**
@@ -385,6 +407,143 @@ class StemmerPatchTrieBinaryIOTest {
} }
} }
/**
* Verifies that stream overload with dense span override delegates to the
* four-argument readFrom method.
*/
@SuppressWarnings("unchecked")
@Test
@DisplayName("Should delegate stream read with dense span override")
void shouldDelegateInputStreamReadWithDenseSpanOverride() throws IOException {
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
final byte[] gzipPayload = gzip("binary-content-with-max-expanded-index");
try (@SuppressWarnings("rawtypes")
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO
.read(new ByteArrayInputStream(gzipPayload), 17);
assertSame(expectedTrie, actualTrie,
"read(InputStream, int) must return the trie produced by FrequencyTrie.readFrom(...).");
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), eq(17)));
}
}
/**
* Verifies that path overload with dense span override delegates to the
* same method overload with the override parameter.
*/
@SuppressWarnings("unchecked")
@Test
@DisplayName("Should delegate path read with dense span override")
void shouldDelegatePathReadWithDenseSpanOverride() throws IOException {
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
final Path sourceFile = temporaryDirectory.resolve("input-max-expanded.bin.gz");
Files.write(sourceFile, gzip("path-based-max-expanded-index"));
try (@SuppressWarnings("rawtypes")
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile, 0);
assertSame(expectedTrie, actualTrie,
"read(Path, int) must return the trie produced by FrequencyTrie.readFrom(...).");
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), eq(0)));
}
}
/**
* Verifies that string path overload with dense span override delegates to the
* same method overload with the override parameter.
*/
@SuppressWarnings("unchecked")
@Test
@DisplayName("Should delegate file name read with dense span override")
void shouldDelegateStringReadWithDenseSpanOverride() throws IOException {
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
final Path sourceFile = temporaryDirectory.resolve("input-string-max-expanded.bin.gz");
Files.write(sourceFile, gzip("string-based-max-expanded-index"));
try (@SuppressWarnings("rawtypes")
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile.toString(), 32);
assertSame(expectedTrie, actualTrie,
"read(String, int) must return the trie produced by FrequencyTrie.readFrom(...).");
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
any(FrequencyTrie.ValueStreamCodec.class), eq(32)));
}
}
/**
* Verifies that metadata-only read parses and returns the persisted metadata.
*/
@Test
@DisplayName("Should read metadata from gzip payload")
void shouldReadMetadataFromGzipPayload() throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("run", PatchCommandEncoder.builder().build().encode("running", "run"));
final FrequencyTrie<String> trie = builder.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
StemmerPatchTrieBinaryIO.write(trie, outputStream);
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(new ByteArrayInputStream(outputStream.toByteArray()));
assertEquals(trie.metadata(), metadata,
"readMetadata(InputStream) must return the same metadata persisted by write().");
}
/**
* Verifies that metadata can be read from a binary file path.
*/
@Test
@DisplayName("Should read metadata from file path")
void shouldReadMetadataFromPath() throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
final FrequencyTrie<String> trie = builder.build();
final Path sourceFile = temporaryDirectory.resolve("metadata-path.bin.gz");
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile);
assertEquals(trie.metadata(), metadata);
}
/**
* Verifies that metadata can be read from a binary file name.
*/
@Test
@DisplayName("Should read metadata from file name")
void shouldReadMetadataFromStringPath() throws IOException {
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
final FrequencyTrie<String> trie = builder.build();
final Path sourceFile = temporaryDirectory.resolve("metadata-string.bin.gz");
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile.toString());
assertEquals(trie.metadata(), metadata);
}
/** /**
* Verifies that malformed non-GZip input is reported as an I/O failure. * Verifies that malformed non-GZip input is reported as an I/O failure.
*/ */
@@ -482,6 +641,10 @@ class StemmerPatchTrieBinaryIOTest {
/** /**
* Output stream that records whether it has been closed. * Output stream that records whether it has been closed.
*/ */
@Tag("unit")
@Tag("io")
@Tag("trie")
@Tag("persistence")
private static final class TrackingOutputStream extends ByteArrayOutputStream { private static final class TrackingOutputStream extends ByteArrayOutputStream {
/** /**
@@ -508,6 +671,10 @@ class StemmerPatchTrieBinaryIOTest {
/** /**
* Input stream that records whether it has been closed. * Input stream that records whether it has been closed.
*/ */
@Tag("unit")
@Tag("io")
@Tag("trie")
@Tag("persistence")
private static final class TrackingInputStream extends ByteArrayInputStream { private static final class TrackingInputStream extends ByteArrayInputStream {
/** /**

View File

@@ -85,9 +85,12 @@ import org.junit.jupiter.params.provider.MethodSource;
* <li>the current bundled language set, including right-to-left metadata</li> * <li>the current bundled language set, including right-to-left metadata</li>
* </ul> * </ul>
*/ */
@Tag("unit")
@Tag("integration") @Tag("integration")
@Tag("stemmer") @Tag("stemmer")
@Tag("io")
@Tag("parser")
@Tag("trie")
@Tag("persistence")
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
final class StemmerPatchTrieLoaderTest { final class StemmerPatchTrieLoaderTest {
@@ -210,36 +213,43 @@ final class StemmerPatchTrieLoaderTest {
Arguments.of("14-load-binary-string", Arguments.of("14-load-binary-string",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null),
StemmerPatchTrieLoader.FILENAME_REQUIRED), StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("15-load-binary-stream", Arguments.of("15-load-binary-path-override",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
"path"),
Arguments.of("16-load-binary-string-override",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("17-load-binary-stream",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null),
"inputStream"), "inputStream"),
Arguments.of("16-save-binary-null-trie-path", Arguments.of("18-save-binary-null-trie-path",
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"), (ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"),
Arguments.of("17-save-binary-null-path", Arguments.of("19-save-binary-null-path",
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"), (ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"),
Arguments.of("18-save-binary-null-trie-string", Arguments.of("20-save-binary-null-trie-string",
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()), (ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()),
"trie"), "trie"),
Arguments.of("19-save-binary-null-string", Arguments.of("21-save-binary-null-string",
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
StemmerPatchTrieLoader.FILENAME_REQUIRED), StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("20-load-language-null-metadata", Arguments.of("22-load-language-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK, (ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
true, (TrieMetadata) null), true, (TrieMetadata) null),
"metadata"), "metadata"),
Arguments.of("21-load-path-null-metadata", Arguments.of("23-load-path-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
"metadata"), "metadata"),
Arguments.of("22-load-string-null-metadata", Arguments.of("24-load-string-null-metadata",
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true, (ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
(TrieMetadata) null), (TrieMetadata) null),
"metadata"), "metadata"),
Arguments.of("23-load-binary-metadata-path-null", Arguments.of("25-load-binary-metadata-path-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"), (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
Arguments.of("24-load-binary-metadata-string-null", Arguments.of("26-load-binary-metadata-string-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
StemmerPatchTrieLoader.FILENAME_REQUIRED), StemmerPatchTrieLoader.FILENAME_REQUIRED),
Arguments.of("25-load-binary-metadata-stream-null", Arguments.of("27-load-binary-metadata-stream-null",
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null), (ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
"inputStream")); "inputStream"));
} }
@@ -258,6 +268,9 @@ final class StemmerPatchTrieLoaderTest {
*/ */
@Nested @Nested
@DisplayName("API contracts") @DisplayName("API contracts")
@Tag("validation")
@Tag("integration")
@Tag("trie")
final class ApiContractTests { final class ApiContractTests {
/** /**
@@ -306,11 +319,83 @@ final class StemmerPatchTrieLoaderTest {
} }
} }
/**
* Focused internal loader behavior tests.
*/
@Nested
@DisplayName("Internal helper behavior")
@Tag("construction")
@Tag("integration")
@Tag("trie")
final class InternalLoaderBehaviorTests {
/**
* Verifies that bundled language loading follows explicit
* right-to-left metadata mapping.
*/
@Test
@DisplayName("bundled language loading must infer traversal direction from language metadata")
void shouldLoadBundledLanguagesUsingLanguageRightToLeftMetadata() throws IOException {
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
final FrequencyTrie<String> leftToRightDictionary = StemmerPatchTrieLoader.load(
StemmerPatchTrieLoader.Language.US_UK, true, settings);
final FrequencyTrie<String> rightToLeftDictionary = StemmerPatchTrieLoader.load(
StemmerPatchTrieLoader.Language.FA_IR, true, settings);
assertEquals(WordTraversalDirection.BACKWARD, leftToRightDictionary.traversalDirection(),
"Left-to-right languages should use backward traversal.");
assertEquals(WordTraversalDirection.FORWARD, rightToLeftDictionary.traversalDirection(),
"Right-to-left languages should use forward traversal.");
}
/**
* Verifies the mode-based and settings-based bundled load overloads remain
* semantically consistent for traversal direction.
*/
@Test
@DisplayName("load(Language,.., ReductionMode) and load(Language,.., ReductionSettings) should agree on traversal direction")
void shouldKeepBundledTraversalDirectionConsistentAcrossOverloads() throws IOException {
final FrequencyTrie<String> byMode = StemmerPatchTrieLoader.load(
StemmerPatchTrieLoader.Language.US_UK, true, DEFAULT_REDUCTION_MODE);
final FrequencyTrie<String> bySettings = StemmerPatchTrieLoader.load(
StemmerPatchTrieLoader.Language.US_UK, true,
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE));
assertEquals(byMode.traversalDirection(), bySettings.traversalDirection());
assertEquals(byMode.metadata().reductionSettings().reductionMode(),
bySettings.metadata().reductionSettings().reductionMode());
}
/**
* Verifies bundled resource access succeeds for known resources and fails
* for unknown resources.
*/
@Test
@DisplayName("openBundledResource should return readable streams for known resources and fail for unknown ones")
void shouldOpenBundledResourcesSuccessfullyAndRejectUnknowns() throws IOException {
final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath();
try (InputStream inputStream = StemmerPatchTrieLoader.openBundledResource(resourcePath);
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
assertNotNull(reader.readLine(), "Known bundled resource must expose readable content.");
}
final String missingResource = "org/egothor/stemmer/missing-dictionary.dict.gz";
final IOException exception = assertThrows(IOException.class,
() -> StemmerPatchTrieLoader.openBundledResource(missingResource));
assertEquals("Stemmer resource not found: " + missingResource, exception.getMessage());
}
}
/** /**
* Focused filesystem and parser behavior tests. * Focused filesystem and parser behavior tests.
*/ */
@Nested @Nested
@DisplayName("Filesystem and parser behavior") @DisplayName("Filesystem and parser behavior")
@Tag("io")
@Tag("construction")
@Tag("integration")
@Tag("trie")
final class FilesystemAndParserTests { final class FilesystemAndParserTests {
/** /**
@@ -512,6 +597,44 @@ final class StemmerPatchTrieLoaderTest {
} }
} }
/**
* Verifies that binary load overloads with an explicit dense lookup span
* preserve trie semantics while honoring the dense-layout override.
*/
@Test
@DisplayName("Binary dense-span override overloads should load equivalent tries")
void shouldLoadBinaryWithDenseSpanOverrideOverloads() throws IOException {
final Path dictionaryFile = writeDictionary("""
run running runs runner
city cities
study studies studying
""");
final Path binaryFile = tempDir.resolve("stemmer-trie-overrides.bin.gz");
final FrequencyTrie<String> original = StemmerPatchTrieLoader.load(dictionaryFile, true,
DEFAULT_REDUCTION_MODE);
StemmerPatchTrieLoader.saveBinary(original, binaryFile);
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX);
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
"studying");
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
"studying");
assertFalse(fromPathNoDense.root().hasDenseLookup(),
"Zero span should disable dense lookup on the loaded root.");
assertFalse(fromStringNoDense.root().hasDenseLookup(),
"Zero span should disable dense lookup on the loaded root.");
}
/** /**
* Writes a dictionary file into the temporary directory. * Writes a dictionary file into the temporary directory.
* *
@@ -530,7 +653,11 @@ final class StemmerPatchTrieLoaderTest {
* Bundled dictionary integration tests. * Bundled dictionary integration tests.
*/ */
@Nested @Nested
@Tag("slow")
@DisplayName("Bundled dictionaries") @DisplayName("Bundled dictionaries")
@Tag("compat")
@Tag("trie")
@Tag("regression")
final class BundledDictionaryTests { final class BundledDictionaryTests {
/** /**

View File

@@ -44,7 +44,7 @@ import java.util.Set;
import net.jqwik.api.ForAll; import net.jqwik.api.ForAll;
import net.jqwik.api.Label; import net.jqwik.api.Label;
import net.jqwik.api.Property; import net.jqwik.api.Property;
import net.jqwik.api.Tag; import org.junit.jupiter.api.Tag;
/** /**
* Property-based tests for patch-command stemmer tries. * Property-based tests for patch-command stemmer tries.
@@ -56,9 +56,8 @@ import net.jqwik.api.Tag;
* persistence must not alter that behavior. * persistence must not alter that behavior.
*/ */
@Label("Stemmer patch trie properties") @Label("Stemmer patch trie properties")
@Tag("unit")
@Tag("property") @Tag("property")
@Tag("stemming") @Tag("stemmer")
class StemmerPatchTrieProperties extends PropertyBasedTestSupport { class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
/** /**

View File

@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@Tag("unit") @Tag("unit")
@Tag("metadata")
@Tag("trie")
@Tag("validation")
@DisplayName("TrieMetadata") @DisplayName("TrieMetadata")
class TrieMetadataTest { class TrieMetadataTest {

View File

@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@Tag("unit") @Tag("unit")
@Tag("core")
@Tag("stemmer")
@Tag("validation")
@DisplayName("WordTraversalDirection") @DisplayName("WordTraversalDirection")
class WordTraversalDirectionTest { class WordTraversalDirectionTest {

View File

@@ -45,7 +45,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link ChildDescriptor}. * Unit tests for {@link ChildDescriptor}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("ChildDescriptor") @DisplayName("ChildDescriptor")
class ChildDescriptorTest { class ChildDescriptorTest {

View File

@@ -31,8 +31,10 @@
package org.egothor.stemmer.trie; package org.egothor.stemmer.trie;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Tag;
@@ -43,11 +45,40 @@ import org.junit.jupiter.api.Test;
* documented backing-array exposure. * documented backing-array exposure.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast")
@Tag("trie") @Tag("trie")
@Tag("lookup")
@DisplayName("CompiledNode and NodeData") @DisplayName("CompiledNode and NodeData")
class CompiledNodeAndNodeDataTest { class CompiledNodeAndNodeDataTest {
/**
* Creates a typed child array for compiled-node tests.
*
* @param length requested array length
* @return typed child array
*/
@SuppressWarnings("unchecked")
private static CompiledNode<String>[] children(final int length) {
return new CompiledNode[length];
}
/**
* Creates an empty child array for leaf compiled-node tests.
*
* @return empty typed child array
*/
private static CompiledNode<String>[] noChildren() {
return children(0);
}
/**
* Creates a leaf node used as a child in lookup tests.
*
* @return leaf node
*/
private static CompiledNode<String> leaf() {
return new CompiledNode<>(new char[0], noChildren(), new String[0], new int[0]);
}
/** /**
* Verifies that {@link NodeData} rejects mismatched edge-related array lengths. * Verifies that {@link NodeData} rejects mismatched edge-related array lengths.
*/ */
@@ -98,8 +129,7 @@ class CompiledNodeAndNodeDataTest {
@Test @Test
@DisplayName("CompiledNode rejects mismatched edge and child arrays") @DisplayName("CompiledNode rejects mismatched edge and child arrays")
void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() { void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() {
@SuppressWarnings("unchecked") final CompiledNode<String>[] children = noChildren();
final CompiledNode<String>[] children = new CompiledNode[0];
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0])); () -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0]));
@@ -113,8 +143,7 @@ class CompiledNodeAndNodeDataTest {
@Test @Test
@DisplayName("CompiledNode rejects mismatched value arrays") @DisplayName("CompiledNode rejects mismatched value arrays")
void compiledNodeShouldRejectMismatchedValueArrays() { void compiledNodeShouldRejectMismatchedValueArrays() {
@SuppressWarnings("unchecked") final CompiledNode<String>[] children = noChildren();
final CompiledNode<String>[] children = new CompiledNode[0];
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
() -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0])); () -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0]));
@@ -130,8 +159,7 @@ class CompiledNodeAndNodeDataTest {
@DisplayName("CompiledNode accessors expose documented backing arrays") @DisplayName("CompiledNode accessors expose documented backing arrays")
void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() { void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() {
final char[] edgeLabels = new char[] { 'a' }; final char[] edgeLabels = new char[] { 'a' };
@SuppressWarnings("unchecked") final CompiledNode<String>[] children = children(1);
final CompiledNode<String>[] children = new CompiledNode[1];
final String[] orderedValues = new String[] { "stem" }; final String[] orderedValues = new String[] { "stem" };
final int[] orderedCounts = new int[] { 5 }; final int[] orderedCounts = new int[] { 5 };
final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts); final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts);
@@ -141,4 +169,130 @@ class CompiledNodeAndNodeDataTest {
assertSame(orderedValues, node.orderedValues()); assertSame(orderedValues, node.orderedValues());
assertSame(orderedCounts, node.orderedCounts()); assertSame(orderedCounts, node.orderedCounts());
} }
/**
* Verifies that dense lookup is used when the interval is compact.
*/
@Test
@DisplayName("CompiledNode can resolve child via dense lookup table")
void compiledNodeUsesDenseLookupForCompactIntervals() {
final CompiledNode<String>[] children = children(4);
children[0] = leaf();
children[1] = leaf();
children[2] = leaf();
children[3] = leaf();
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'b', 'c', 'd' }, children,
new String[] { "1", "2", "3", "4" }, new int[] { 1, 1, 1, 1 });
assertTrue(node.hasDenseLookup());
assertSame(children[0], node.findChild('a'));
assertSame(children[3], node.findChild('d'));
assertSame(null, node.findChild('z'));
}
/**
* Verifies that fallback linear scan is used for small node degree.
*/
@Test
@DisplayName("CompiledNode resolves child by linear scan for small degree")
void compiledNodeUsesLinearScanForSmallDegree() {
final CompiledNode<String>[] children = children(4);
final CompiledNode<String> childA = leaf();
final CompiledNode<String> childB = leaf();
final CompiledNode<String> childC = leaf();
final CompiledNode<String> childD = leaf();
children[0] = childA;
children[1] = childB;
children[2] = childC;
children[3] = childD;
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'z', '中', '你' }, children,
new String[] { "1", "2", "3", "4" }, 0, new int[] { 1, 1, 1, 1 });
assertFalse(node.hasDenseLookup());
assertSame(childA, node.findChild('a'));
assertSame(childD, node.findChild('你'));
assertSame(null, node.findChild('b'));
}
/**
* Verifies that fallback binary search is used for larger node degree without
* dense lookup.
*/
@Test
@DisplayName("CompiledNode resolves child by binary search for large degree")
void compiledNodeUsesBinarySearchForLargeDegree() {
final CompiledNode<String>[] children = children(5);
final CompiledNode<String> childA = leaf();
final CompiledNode<String> childB = leaf();
final CompiledNode<String> childC = leaf();
final CompiledNode<String> childD = leaf();
final CompiledNode<String> childE = leaf();
children[0] = childA;
children[1] = childB;
children[2] = childC;
children[3] = childD;
children[4] = childE;
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'c', 'k', 't', 'z' }, children,
new String[] { "1", "2", "3", "4", "5" }, 0, new int[] { 1, 1, 1, 1, 1 });
assertFalse(node.hasDenseLookup());
assertSame(childC, node.findChild('k'));
assertSame(childE, node.findChild('z'));
assertSame(null, node.findChild('x'));
}
/**
* Verifies the basic node-state helpers that are used by diagnostics and
* behavioral checks.
*/
@Test
@DisplayName("CompiledNode reports leaf, value and edge presence state")
void compiledNodeReportsNodeStateHelpers() {
final CompiledNode<String>[] childless = noChildren();
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], childless, new String[0], new int[0]);
assertTrue(leaf.isLeaf());
assertFalse(leaf.hasChildren());
assertFalse(leaf.hasValues());
assertFalse(leaf.hasEdge('a'));
final CompiledNode<String>[] child = children(1);
final String[] orderedValues = new String[] { "leaf" };
final int[] orderedCounts = new int[] { 1 };
child[0] = new CompiledNode<>(new char[0], noChildren(), orderedValues, orderedCounts);
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a' }, child, orderedValues, orderedCounts);
assertFalse(node.isLeaf());
assertTrue(node.hasChildren());
assertTrue(node.hasValues());
assertTrue(node.valueCount() > 0);
assertTrue(node.hasEdge('a'));
assertFalse(node.hasEdge('b'));
}
/**
* Verifies structural equality and hash-code behavior for compiled nodes.
*/
@Test
@DisplayName("CompiledNode equals and hashCode align for identical structure")
void compiledNodeEqualsAndHashCodeAlignForIdenticalStructure() {
final CompiledNode<String>[] child = children(1);
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], noChildren(), new String[] { "v" },
new int[] { 1 });
child[0] = leaf;
final CompiledNode<String> first = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
new int[] { 2 });
final CompiledNode<String> second = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
new int[] { 2 });
assertEquals(first, second);
assertEquals(first.hashCode(), second.hashCode());
}
} }

View File

@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link DominantLocalDescriptor}. * Unit tests for {@link DominantLocalDescriptor}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("DominantLocalDescriptor") @DisplayName("DominantLocalDescriptor")
class DominantLocalDescriptorTest { class DominantLocalDescriptorTest {

View File

@@ -50,7 +50,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link LocalValueSummary}. * Unit tests for {@link LocalValueSummary}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("LocalValueSummary") @DisplayName("LocalValueSummary")
class LocalValueSummaryTest { class LocalValueSummaryTest {
@@ -190,6 +191,9 @@ class LocalValueSummaryTest {
/** /**
* Test helper with identical textual form but distinct identity. * Test helper with identical textual form but distinct identity.
*/ */
@Tag("unit")
@Tag("trie")
@Tag("reduction")
private static final class TextTwin { private static final class TextTwin {
/** /**

View File

@@ -44,7 +44,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link MutableNode}. * Unit tests for {@link MutableNode}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("construction")
@DisplayName("MutableNode") @DisplayName("MutableNode")
class MutableNodeTest { class MutableNodeTest {

View File

@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link RankedLocalDescriptor}. * Unit tests for {@link RankedLocalDescriptor}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("RankedLocalDescriptor") @DisplayName("RankedLocalDescriptor")
class RankedLocalDescriptorTest { class RankedLocalDescriptorTest {

View File

@@ -48,7 +48,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link ReducedNode}. * Unit tests for {@link ReducedNode}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("ReducedNode") @DisplayName("ReducedNode")
class ReducedNodeTest { class ReducedNodeTest {

View File

@@ -47,7 +47,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link ReductionContext}. * Unit tests for {@link ReductionContext}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("ReductionContext") @DisplayName("ReductionContext")
class ReductionContextTest { class ReductionContextTest {

View File

@@ -46,7 +46,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link ReductionSignature}. * Unit tests for {@link ReductionSignature}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("ReductionSignature") @DisplayName("ReductionSignature")
class ReductionSignatureTest { class ReductionSignatureTest {

View File

@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
* Unit tests for {@link UnorderedLocalDescriptor}. * Unit tests for {@link UnorderedLocalDescriptor}.
*/ */
@Tag("unit") @Tag("unit")
@Tag("fast") @Tag("trie")
@Tag("reduction")
@DisplayName("UnorderedLocalDescriptor") @DisplayName("UnorderedLocalDescriptor")
class UnorderedLocalDescriptorTest { class UnorderedLocalDescriptorTest {