Compare commits
6 Commits
release@2.
...
release@2.
| Author | SHA1 | Date | |
|---|---|---|---|
|
b945902f05
|
|||
|
902ad117e8
|
|||
|
14a1e2fc53
|
|||
|
87ff85fd6d
|
|||
|
7bd0fc66ba
|
|||
|
dadab5514e
|
12
.github/workflows/build.yml
vendored
12
.github/workflows/build.yml
vendored
@@ -51,7 +51,7 @@ jobs:
|
||||
test -f gradle/verification-metadata.xml
|
||||
|
||||
- name: Execute build, tests, PMD, coverage, Javadoc, distribution packaging, and SBOM generation
|
||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport distZip cyclonedxBom
|
||||
run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom
|
||||
|
||||
- name: Upload SBOM
|
||||
if: always()
|
||||
@@ -70,8 +70,8 @@ jobs:
|
||||
with:
|
||||
name: test-reports
|
||||
path: |
|
||||
build/reports/tests/test
|
||||
build/test-results/test
|
||||
build/reports/tests
|
||||
build/test-results
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -90,8 +90,8 @@ jobs:
|
||||
with:
|
||||
name: coverage-reports
|
||||
path: |
|
||||
build/reports/jacoco/test/html
|
||||
build/reports/jacoco/test/jacocoTestReport.xml
|
||||
build/reports/jacoco/jacocoCiReleaseReport/html
|
||||
build/reports/jacoco/jacocoCiReleaseReport/jacocoCiReleaseReport.xml
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
env:
|
||||
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
|
||||
SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
|
||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport cyclonedxBom centralBundle
|
||||
run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom centralBundle
|
||||
|
||||
- name: Generate release changelog
|
||||
shell: bash
|
||||
|
||||
19
.github/workflows/pages.yml
vendored
19
.github/workflows/pages.yml
vendored
@@ -70,7 +70,7 @@ jobs:
|
||||
test -f gradle/verification-metadata.xml
|
||||
|
||||
- name: Build reports for publication
|
||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport pitest jmh cyclonedxBom
|
||||
run: ./gradlew --no-daemon clean ciRelease pmdMain javadoc jacocoCiReleaseReport pitest jmh cyclonedxBom
|
||||
|
||||
- name: Prepare gh-pages worktree
|
||||
shell: bash
|
||||
@@ -93,6 +93,9 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TEST_REPORT_DIR="build/reports/tests/ciRelease"
|
||||
JACOCO_REPORT_DIR="build/reports/jacoco/jacocoCiReleaseReport"
|
||||
|
||||
SITE_DIR=".gh-pages"
|
||||
RUN_DIR="${SITE_DIR}/builds/${GITHUB_RUN_NUMBER}"
|
||||
RUN_METRICS_DIR="${RUN_DIR}/metrics"
|
||||
@@ -106,14 +109,14 @@ jobs:
|
||||
cp -R build/docs/javadoc "${RUN_DIR}/javadoc"
|
||||
cp -R build/docs/javadoc "${LATEST_DIR}/javadoc"
|
||||
|
||||
cp -R build/reports/tests/test "${RUN_DIR}/test"
|
||||
cp -R build/reports/tests/test "${LATEST_DIR}/test"
|
||||
cp -R "${TEST_REPORT_DIR}" "${RUN_DIR}/test"
|
||||
cp -R "${TEST_REPORT_DIR}" "${LATEST_DIR}/test"
|
||||
|
||||
cp -R build/reports/pmd "${RUN_DIR}/pmd"
|
||||
cp -R build/reports/pmd "${LATEST_DIR}/pmd"
|
||||
|
||||
cp -R build/reports/jacoco/test/html "${RUN_DIR}/coverage"
|
||||
cp -R build/reports/jacoco/test/html "${LATEST_DIR}/coverage"
|
||||
cp -R "${JACOCO_REPORT_DIR}/html" "${RUN_DIR}/coverage"
|
||||
cp -R "${JACOCO_REPORT_DIR}/html" "${LATEST_DIR}/coverage"
|
||||
|
||||
cp -R build/reports/pitest "${RUN_DIR}/pitest"
|
||||
cp -R build/reports/pitest "${LATEST_DIR}/pitest"
|
||||
@@ -178,7 +181,7 @@ jobs:
|
||||
|
||||
python3 \
|
||||
./tools/generate-pages-badges.py \
|
||||
--jacoco-xml build/reports/jacoco/test/jacocoTestReport.xml \
|
||||
--jacoco-xml "${JACOCO_REPORT_DIR}/jacocoCiReleaseReport.xml" \
|
||||
--pit-xml build/reports/pitest/mutations.xml \
|
||||
--jmh-csv build/reports/jmh/jmh-results.csv \
|
||||
--run-metrics-dir "${RUN_METRICS_DIR}" \
|
||||
@@ -228,7 +231,7 @@ jobs:
|
||||
<p class="meta">Build ${GITHUB_RUN_NUMBER} from commit ${GITHUB_SHA}</p>
|
||||
<ul>
|
||||
<li><a href="./javadoc/">Javadoc</a></li>
|
||||
<li><a href="./test/">Test Report</a></li>
|
||||
<li><a href="./test/">Release Verification Test Report (ciRelease)</a></li>
|
||||
<li><a href="./pmd/main.html">PMD Report</a></li>
|
||||
<li><a href="./coverage/">Coverage Report</a></li>
|
||||
${DEPENDENCY_CHECK_LINK:-<li>Dependency Vulnerability Report: not available</li>}
|
||||
@@ -260,7 +263,7 @@ jobs:
|
||||
|
||||
- [Latest build summary](https://leogalambos.github.io/Radixor/builds/latest/)
|
||||
- [Javadoc](https://leogalambos.github.io/Radixor/builds/latest/javadoc/)
|
||||
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||
- [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
||||
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||
|
||||
27
.project
27
.project
@@ -2,21 +2,22 @@
|
||||
<projectDescription>
|
||||
<name>Radixor</name>
|
||||
<comment></comment>
|
||||
<projects/>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
||||
</natures>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<linkedResources/>
|
||||
<filteredResources/>
|
||||
</projectDescription>
|
||||
|
||||
19
README.md
19
README.md
@@ -1,15 +1,13 @@
|
||||
<img src="Radixor.png" width="30%" align="right" alt="Radixor logo" />
|
||||
<img src="docs/assets/images/banner.jpg" width="100%" alt="Radixor banner" />
|
||||
|
||||
# Radixor
|
||||
|
||||
[](https://github.com/leogalambos/Radixor/actions/workflows/build.yml)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt)
|
||||
[](https://central.sonatype.com/artifact/org.egothor/radixor)
|
||||
[](LICENSE)
|
||||
[](#)
|
||||
[](https://central.sonatype.com/artifact/org.egothor/radixor)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/)
|
||||
[](https://github.com/leogalambos/Radixor/actions/workflows/build.yml)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||
[](https://leogalambos.github.io/Radixor/builds/latest/jmh/jmh-results.txt)
|
||||
|
||||
*Fast, deterministic, multi-language stemming for Java, built around compact patch-command tries and measured at roughly 4× to 6× the throughput of the Snowball Porter stemmer family on the current English benchmark workload.*
|
||||
|
||||
@@ -167,6 +165,9 @@ The repository keeps the front page concise and places detailed documentation un
|
||||
- [Architecture](docs/architecture.md)
|
||||
Structural model, data flow, and runtime lookup behavior.
|
||||
|
||||
- [Lookup Edge Optimization](docs/lookup-edge-optimization.md)
|
||||
Speed/memory trade-off of dense child edge lookup in compiled tries.
|
||||
|
||||
- [Reduction Semantics](docs/reduction-semantics.md)
|
||||
Ranked, unordered, and dominant reduction behavior.
|
||||
|
||||
|
||||
161
build.gradle
161
build.gradle
@@ -108,9 +108,19 @@ dependencyCheck {
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType(Test).configureEach {
|
||||
useJUnitPlatform()
|
||||
def cliIncludeTags = project.findProperty('includeTags')?.toString() ?: System.getProperty('includeTags')
|
||||
def cliExcludeTags = project.findProperty('excludeTags')?.toString() ?: System.getProperty('excludeTags')
|
||||
|
||||
def splitTagExpression = { String tagsExpr ->
|
||||
if (tagsExpr == null || tagsExpr.isBlank()) {
|
||||
return []
|
||||
}
|
||||
return tagsExpr.split(',')
|
||||
.collect { it.trim() }
|
||||
.findAll { it != null && !it.isBlank() }
|
||||
}
|
||||
|
||||
tasks.withType(Test).configureEach {
|
||||
doFirst {
|
||||
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
||||
}
|
||||
@@ -123,14 +133,127 @@ tasks.withType(Test).configureEach {
|
||||
minHeapSize = '1g'
|
||||
maxHeapSize = '4g'
|
||||
|
||||
finalizedBy(tasks.named('jacocoTestReport'))
|
||||
|
||||
reports {
|
||||
junitXml.required = true
|
||||
html.required = true
|
||||
}
|
||||
}
|
||||
|
||||
def configureJUnitPlatformTags = { Test task, String includeTagsExpr, String excludeTagsExpr ->
|
||||
task.useJUnitPlatform {
|
||||
final def includes = splitTagExpression(includeTagsExpr)
|
||||
final def excludes = splitTagExpression(excludeTagsExpr)
|
||||
|
||||
if (!includes.isEmpty()) {
|
||||
includeTags(*includes.toArray(new String[0]))
|
||||
}
|
||||
if (!excludes.isEmpty()) {
|
||||
excludeTags(*excludes.toArray(new String[0]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.named('test', Test) {
|
||||
configureJUnitPlatformTags(it, cliIncludeTags, cliExcludeTags)
|
||||
finalizedBy(tasks.named('jacocoTestReport'))
|
||||
}
|
||||
|
||||
def configureTaggedTestProfile = { String taskName, String includeTagsExpr, String excludeTagsExpr = null,
|
||||
String taskDescription = null, String testNameExcludePatterns = null ->
|
||||
tasks.register(taskName, Test) {
|
||||
group = 'verification'
|
||||
description = taskDescription
|
||||
|
||||
configureJUnitPlatformTags(delegate as Test, includeTagsExpr, excludeTagsExpr)
|
||||
testClassesDirs = sourceSets.test.output.classesDirs
|
||||
classpath = sourceSets.test.runtimeClasspath
|
||||
dependsOn(tasks.named('compileTestJava'))
|
||||
|
||||
doFirst {
|
||||
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
||||
}
|
||||
|
||||
if (testNameExcludePatterns != null && !testNameExcludePatterns.isBlank()) {
|
||||
filter {
|
||||
testNameExcludePatterns.split(',').each { String pattern ->
|
||||
final def trimmedPattern = pattern.trim()
|
||||
if (!trimmedPattern.isEmpty()) {
|
||||
excludeTestsMatching(trimmedPattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
minHeapSize = '1g'
|
||||
maxHeapSize = '4g'
|
||||
|
||||
reports {
|
||||
junitXml.required = true
|
||||
html.required = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciSmoke',
|
||||
'unit',
|
||||
'slow',
|
||||
'Fast feedback profile for unit tests with slow tests explicitly excluded.',
|
||||
'org.egothor.stemmer.CompileIntegrationTest*'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciCore',
|
||||
'unit,trie,frequency-trie,property',
|
||||
null,
|
||||
'Focused profile for core trie behavior and trie-specific property checks.'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciIntegration',
|
||||
'integration',
|
||||
'slow',
|
||||
'Integration pipeline profile (loader/parser/CLI/IO end-to-end flows) excluding slow integration paths.'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciSlow',
|
||||
'slow',
|
||||
null,
|
||||
'Targeted profile for all slow tests (large dictionaries, long-running corpus validation, and heavy integration checks).'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciCompat',
|
||||
'compat,regression',
|
||||
null,
|
||||
'Compatibility profile guarding persisted artifact and compatibility regressions.'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciRelease',
|
||||
null,
|
||||
'slow',
|
||||
'Release-profile validation of all non-slow tests.',
|
||||
'org.egothor.stemmer.CompileIntegrationTest*,org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*'
|
||||
)
|
||||
|
||||
configureTaggedTestProfile(
|
||||
'ciNightly',
|
||||
'fuzz',
|
||||
null,
|
||||
'Nightly robustness profile with fuzz testing emphasis.'
|
||||
)
|
||||
|
||||
tasks.register('ci') {
|
||||
group = 'verification'
|
||||
description = 'Runs the full enterprise CI profile set in sequence.'
|
||||
dependsOn(tasks.named('ciSmoke'))
|
||||
dependsOn(tasks.named('ciCore'))
|
||||
dependsOn(tasks.named('ciIntegration'))
|
||||
dependsOn(tasks.named('ciCompat'))
|
||||
}
|
||||
|
||||
tasks.withType(Pmd).configureEach {
|
||||
reports {
|
||||
xml.required = true
|
||||
@@ -155,6 +278,36 @@ tasks.named('jacocoTestReport', JacocoReport) {
|
||||
}
|
||||
}
|
||||
|
||||
def registerJacocoProfileReport = { String reportTaskName, String sourceTaskName ->
|
||||
tasks.register(reportTaskName, JacocoReport) {
|
||||
group = 'verification'
|
||||
description = "Generates Jacoco report for ${sourceTaskName} execution."
|
||||
|
||||
dependsOn(tasks.named(sourceTaskName))
|
||||
|
||||
classDirectories.setFrom(
|
||||
files(sourceSets.main.output).asFileTree.matching {
|
||||
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
|
||||
exclude 'org/egothor/stemmer/DiacriticStripper*'
|
||||
}
|
||||
)
|
||||
|
||||
executionData.setFrom(
|
||||
fileTree(layout.buildDirectory.dir('jacoco')) {
|
||||
include "${sourceTaskName}.exec"
|
||||
}
|
||||
)
|
||||
|
||||
reports {
|
||||
xml.required = true
|
||||
csv.required = false
|
||||
html.required = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
registerJacocoProfileReport('jacocoCiReleaseReport', 'ciRelease')
|
||||
|
||||
tasks.named('check') {
|
||||
dependsOn(tasks.named('jacocoTestReport'))
|
||||
// no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze'))
|
||||
|
||||
@@ -75,6 +75,14 @@ The distinction between preferred-result lookup and multi-result lookup is part
|
||||
|
||||
That model is part of how the public API should be understood.
|
||||
|
||||
Visitor lookup methods such as `getAllNormalized(..., EntrySink, maxResults)` are additive hot-path APIs. They expose the same local ordering and count semantics without allocating result containers, but they do not replace `get()`, `getAll()`, or `getEntries()`.
|
||||
|
||||
Compiled `FrequencyTrie` instances are immutable and thread-safe for concurrent reads. Visitor sinks are caller-owned and are not retained by the trie. Stored values passed to sinks are the model-owned trie values; for `FrequencyTrie<String>` patch tries, those patch strings are immutable stored strings rather than fresh per-result strings.
|
||||
|
||||
### Stable patch application behavior
|
||||
|
||||
`PatchCommandEncoder.apply(...)` remains the compatibility API for string-returning patch application. Buffer-oriented `applyTo(...)` overloads are additive APIs for caller-owned output storage. They do not retain output arrays, report insufficient capacity with `APPLY_INSUFFICIENT_CAPACITY`, and preserve the existing malformed-patch compatibility behavior where `apply(...)` preserves the source.
|
||||
|
||||
### Stable reduction-mode intent
|
||||
|
||||
Each public `ReductionMode` constant carries a semantic contract that should remain meaningful across versions.
|
||||
|
||||
193
docs/lookup-edge-optimization.md
Normal file
193
docs/lookup-edge-optimization.md
Normal file
@@ -0,0 +1,193 @@
|
||||
# Lookup Edge Optimization
|
||||
|
||||
Compiled trie nodes (`CompiledNode`) use three lookup strategies when resolving child edges:
|
||||
|
||||
1. dense array direct lookup,
|
||||
2. linear scan for very small child counts,
|
||||
3. binary search over sorted edge labels.
|
||||
|
||||
This page explains the dense path, what `maxExpandedIndex` controls, and how to tune it.
|
||||
|
||||
## Runtime model of one node
|
||||
|
||||
For a node with sorted edge labels `char[] edges`, the implementation can materialize an
|
||||
index-aligned dense table when labels occupy a small compact code-point interval:
|
||||
|
||||
```text
|
||||
span = maxEdge - minEdge
|
||||
use dense table iff (span <= maxExpandedIndex) and (maxExpandedIndex > 0)
|
||||
```
|
||||
|
||||
When dense lookup is used, lookup is constant-time indexing:
|
||||
|
||||
```text
|
||||
denseIndex = requestedEdge - minEdge
|
||||
return denseChildren[denseIndex] // or null if outside interval
|
||||
```
|
||||
|
||||
When dense lookup is not active (interval is too wide or the configured
|
||||
`maxExpandedIndex` is `0`), `CompiledNode` still chooses between two fallback
|
||||
strategies:
|
||||
|
||||
- **linear scan** for very small child counts (`4` or fewer children),
|
||||
- **binary search** for larger child counts.
|
||||
|
||||
This means the fallback method is selected by child count, not by “distance” alone.
|
||||
`linear scan` is therefore used when there are only a few edges even if those edges are
|
||||
spread across very distant code points.
|
||||
|
||||
### Example: few edges, wide Unicode span
|
||||
|
||||
```text
|
||||
edges = ['a', '中', '你']
|
||||
edge count = 3
|
||||
minEdge = 'a' (U+0061)
|
||||
maxEdge = '你' (U+4F60)
|
||||
span = 20319
|
||||
```
|
||||
|
||||
- If `maxExpandedIndex = 512`, dense indexing is not used because `span > maxExpandedIndex`.
|
||||
- Because `edge count = 3` (<= 4), lookup falls back to a tiny linear scan of the
|
||||
three labels.
|
||||
- This is exactly the case where you get benefit from the threshold even though the interval is wide.
|
||||
|
||||
This is useful for non-Latin scripts as well: what matters is interval width in Unicode
|
||||
code points, not script name. A compact Arabic-range block can still benefit from dense
|
||||
lookups when keys stay in a tight code-point interval.
|
||||
|
||||
## Why this is configurable
|
||||
|
||||
`maxExpandedIndex` is only a performance/paging choice:
|
||||
|
||||
- higher value:
|
||||
- more compact intervals qualify for dense tables,
|
||||
- more constant-time child lookup,
|
||||
- more memory for dense tables in qualifying nodes.
|
||||
- lower value (or `0`):
|
||||
- less dense-table allocation,
|
||||
- fewer branches into constant-time path,
|
||||
- lower materialization memory.
|
||||
|
||||
The value never changes lookup semantics. It only changes the in-memory structure shape.
|
||||
|
||||
## Persistence and loading model
|
||||
|
||||
This threshold is **not** stored in `TrieMetadata`.
|
||||
|
||||
- The binary format stores only trie payload and semantic metadata (`reduction`, `traversal`,
|
||||
case/diacritic settings, and stream version).
|
||||
- `maxExpandedIndex` is chosen when materializing nodes in memory.
|
||||
- You can therefore keep one persisted artifact and load it with different in-memory
|
||||
trade-offs depending on deployment constraints.
|
||||
|
||||
## Default
|
||||
|
||||
- `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX == 512`
|
||||
- `CompiledNode.DEFAULT_MAX_EXPANDED_INDEX == 512`
|
||||
|
||||
These are practical defaults for mixed-language text and Latin-like scripts where edge labels
|
||||
often cluster.
|
||||
|
||||
## Tune during build (writable phase)
|
||||
|
||||
Use the full `FrequencyTrie.Builder` constructor when you are compiling from source data.
|
||||
The builder threshold is applied while freezing reduced nodes into the immutable form.
|
||||
|
||||
```java
|
||||
import org.egothor.stemmer.CaseProcessingMode;
|
||||
import org.egothor.stemmer.DiacriticProcessingMode;
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.ReductionMode;
|
||||
import org.egothor.stemmer.ReductionSettings;
|
||||
import org.egothor.stemmer.WordTraversalDirection;
|
||||
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
final FrequencyTrie.Builder<String> fastBuilder =
|
||||
new FrequencyTrie.Builder<>(String[]::new,
|
||||
settings,
|
||||
WordTraversalDirection.BACKWARD,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
DiacriticProcessingMode.AS_IS,
|
||||
1024); // prefer lookup speed
|
||||
|
||||
// ... put(...) ...
|
||||
final FrequencyTrie<String> trie = fastBuilder.build();
|
||||
```
|
||||
|
||||
Use `0` or `256` for lower memory while still building larger tries.
|
||||
|
||||
```java
|
||||
final FrequencyTrie.Builder<String> compactBuilder =
|
||||
new FrequencyTrie.Builder<>(String[]::new,
|
||||
settings,
|
||||
WordTraversalDirection.BACKWARD,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
DiacriticProcessingMode.AS_IS,
|
||||
256); // lower memory profile
|
||||
```
|
||||
|
||||
## Tune when loading a binary artifact (runtime phase)
|
||||
|
||||
At artifact load time, you can tune the same trade-off independently of persisted metadata.
|
||||
|
||||
```java
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
var defaultLookup = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"));
|
||||
|
||||
var fastLookup = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"), 1024);
|
||||
|
||||
var compactLookup = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"), 0);
|
||||
```
|
||||
|
||||
You can also set the threshold directly with `FrequencyTrie.readFrom(...)` when reading streams:
|
||||
|
||||
```java
|
||||
import java.io.DataInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
|
||||
public final class StreamLoadExample {
|
||||
|
||||
private StreamLoadExample() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
try (InputStream fileInput = Files.newInputStream(Path.of("stemmers", "english.radixor.gz"));
|
||||
GZIPInputStream gzip = new GZIPInputStream(fileInput);
|
||||
DataInputStream dataInput = new DataInputStream(gzip)) {
|
||||
final FrequencyTrie<String> compactOnLoad = FrequencyTrie.readFrom(
|
||||
dataInput,
|
||||
String[]::new,
|
||||
input -> input.readUTF(),
|
||||
256);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note: the string codec is intentionally inline in this snippet to keep it self-contained.
|
||||
|
||||
## Practical guidance
|
||||
|
||||
- Start with default (`512`) in production and profile before changing it.
|
||||
- Use `0` when memory is the priority and query throughput is not the bottleneck.
|
||||
- Use values around `1024` for workloads dominated by compact alphabets and very hot lookups.
|
||||
|
||||
Trade-off expectation:
|
||||
|
||||
- increasing `maxExpandedIndex` improves lookup speed when edges tend to occupy short spans,
|
||||
- decreasing it reduces per-node auxiliary memory in dense-span nodes.
|
||||
@@ -87,6 +87,43 @@ public final class LoadBinaryExample {
|
||||
|
||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
|
||||
|
||||
## Tune child lookup density when loading binaries
|
||||
|
||||
To optimize hot-path latency, you can tune direct child indexing by passing `maxExpandedIndex`
|
||||
at load time. This does not change persisted metadata, only the materialized in-memory form.
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
public final class LoadBinaryWithDenseLookupExample {
|
||||
|
||||
private LoadBinaryWithDenseLookupExample() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> balanced = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"));
|
||||
|
||||
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"),
|
||||
1024);
|
||||
|
||||
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"),
|
||||
0);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Negative values still use `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX`.
|
||||
|
||||
[Lookup Edge Optimization](lookup-edge-optimization.md) describes the trade-off in detail and examples for build-time tuning as well.
|
||||
|
||||
## Build directly with a mutable builder
|
||||
|
||||
A `FrequencyTrie.Builder<V>` accepts repeated `put(key, value)` calls and compiles the final read-only trie through `build()`. Compilation performs bottom-up reduction and produces the compact immutable runtime representation.
|
||||
|
||||
@@ -33,6 +33,28 @@ import org.egothor.stemmer.ValueCount;
|
||||
final List<ValueCount<String>> entries = trie.getEntries("axes");
|
||||
```
|
||||
|
||||
### Visitor lookup for hot paths
|
||||
|
||||
For allocation-sensitive token loops, use the visitor-style lookup methods. They visit the same ordered local values and counts without allocating a result array, list, or `ValueCount` objects.
|
||||
|
||||
```java
|
||||
trie.getAll("axes", (patch, count, rank) -> {
|
||||
// rank is zero-based and follows the same ordering as getAll(String).
|
||||
return true; // return false to stop after this callback
|
||||
}, 8);
|
||||
```
|
||||
|
||||
If the caller has already normalized the input exactly as required by `trie.metadata()`, the normalized methods avoid lookup normalization buffers too:
|
||||
|
||||
```java
|
||||
final char[] token = "axes".toCharArray();
|
||||
trie.getAllNormalized(token, 0, token.length, (patch, count, rank) -> {
|
||||
return true;
|
||||
}, 8);
|
||||
```
|
||||
|
||||
`getAllNormalized(...)` bypasses `caseProcessingMode` and `diacriticProcessingMode`; callers are responsible for supplying canonical input. `maxResults == 0` visits nothing, negative values are rejected, and a sink returning `false` stops iteration after the current callback.
|
||||
|
||||
## Apply patch commands
|
||||
|
||||
A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch.
|
||||
@@ -45,6 +67,25 @@ final String patch = trie.get(word);
|
||||
final String stem = PatchCommandEncoder.apply(word, patch);
|
||||
```
|
||||
|
||||
Hot paths can apply a patch into caller-owned character storage:
|
||||
|
||||
```java
|
||||
final char[] output = new char[32];
|
||||
final int produced = PatchCommandEncoder.applyTo(
|
||||
word,
|
||||
patch,
|
||||
trie.traversalDirection(),
|
||||
output,
|
||||
0,
|
||||
output.length);
|
||||
|
||||
if (produced != PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
|
||||
final String stem = new String(output, 0, produced);
|
||||
}
|
||||
```
|
||||
|
||||
`applyTo(...)` returns the produced character count on success and `APPLY_INSUFFICIENT_CAPACITY` when the output range is too small. Capacity failure does not write partial output. The source and output ranges of the `char[]` overload must not overlap.
|
||||
|
||||
For multiple candidates:
|
||||
|
||||
```java
|
||||
|
||||
@@ -25,6 +25,7 @@ This is why Radixor can generalize beyond explicitly listed forms and why compil
|
||||
The programmatic API is easier to understand when split by developer task:
|
||||
|
||||
- [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage.
|
||||
- [Lookup Edge Optimization](lookup-edge-optimization.md) explains dense child lookup tuning and the speed/memory trade-off when materializing compiled tries.
|
||||
- [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes.
|
||||
- [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) explains how to reopen compiled tries, add new lexical data, rebuild them, and store them as binary artifacts.
|
||||
|
||||
@@ -40,6 +41,24 @@ The main types involved in programmatic usage are:
|
||||
- `FrequencyTrieBuilders` for reconstructing a mutable builder from a compiled trie,
|
||||
- `ReductionMode` and `ReductionSettings` for controlling compilation semantics.
|
||||
|
||||
## Java module system (JPMS)
|
||||
|
||||
The core artifact is published as an explicit JPMS module:
|
||||
|
||||
```java
|
||||
module org.egothor.radixor;
|
||||
```
|
||||
|
||||
A named consuming module uses:
|
||||
|
||||
```java
|
||||
module example.consumer {
|
||||
requires org.egothor.radixor;
|
||||
}
|
||||
```
|
||||
|
||||
The core module is standalone and can be consumed directly as a normal Java module.
|
||||
|
||||
## Recommended reading order
|
||||
|
||||
For most developers, the best order is:
|
||||
|
||||
@@ -58,6 +58,27 @@ A deterministic system is easier to test, easier to reason about, and safer to i
|
||||
|
||||
The project is intended to maintain very high confidence in both core correctness and behavioral stability.
|
||||
|
||||
The recommended execution strategy is defined by the tagged test profiles in [Test taxonomy and execution filtering](test-taxonomy-and-filtering.md). In practice, teams can execute profile tasks directly:
|
||||
|
||||
- `./gradlew ciSmoke`: fast local/PR safety checks (`unit`, excluding `slow`; additionally excludes
|
||||
`CompileIntegrationTest` as a defensive safeguard).
|
||||
- `./gradlew ciSlow`: enterprise heavy gate for all tests marked with `slow` (typically
|
||||
production dictionary and large corpus verification). This should be used for scheduled/manual
|
||||
hardening gates and not in standard release build.
|
||||
- `./gradlew ciCore`: behavioral coverage of trie and frequency-trie paths (`unit` + `property` where applicable)
|
||||
- `./gradlew ciIntegration`: pipeline and CLI integration path checks
|
||||
- `./gradlew ciCompat`: compatibility and regression verification for persisted artifacts
|
||||
- `./gradlew ciRelease`: full non-slow suite for release-confidence runs (all test tags except `slow`,
|
||||
plus explicit name-based exclusion of `CompileIntegrationTest*` and
|
||||
`StemmerPatchTrieLoaderTest$BundledDictionaryTests*` as additional guardrails)
|
||||
- `./gradlew ciNightly`: extended fuzz profile for robustness hardening
|
||||
- `./gradlew ci`: umbrella profile depending on smoke/core/integration/compat
|
||||
|
||||
## Test taxonomy and execution filtering
|
||||
|
||||
The full tag taxonomy and executable filter examples are documented in
|
||||
[Test taxonomy and execution filtering](test-taxonomy-and-filtering.md).
|
||||
|
||||
### Structural coverage
|
||||
|
||||
High code coverage is treated as a useful signal, but not as a sufficient goal on its own. Coverage is valuable only when the covered scenarios actually pressure the implementation in meaningful ways.
|
||||
|
||||
@@ -67,6 +67,36 @@ public final class LoadBinaryStemmerExample {
|
||||
}
|
||||
```
|
||||
|
||||
You can tune in-memory child lookup density at load time without changing the artifact:
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||
|
||||
public final class LoadBinaryStemmerExampleTuned {
|
||||
|
||||
private LoadBinaryStemmerExampleTuned() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"),
|
||||
1024);
|
||||
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
|
||||
Path.of("stemmers", "english.radixor.gz"),
|
||||
128);
|
||||
|
||||
System.out.println("fast=" + fast.size() + ", compact=" + compact.size());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For the trade-off details, see [Lookup Edge Optimization](lookup-edge-optimization.md).
|
||||
|
||||
### Build or extend a stemmer from dictionary data
|
||||
|
||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.
|
||||
|
||||
@@ -23,7 +23,7 @@ These reports are primarily useful when reviewing the published API surface and
|
||||
|
||||
These reports describe the outcome of core verification and static-analysis stages for the latest published build:
|
||||
|
||||
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||
- [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
||||
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||
|
||||
216
docs/test-taxonomy-and-filtering.md
Normal file
216
docs/test-taxonomy-and-filtering.md
Normal file
@@ -0,0 +1,216 @@
|
||||
# Test Tag Taxonomy and Execution Guide
|
||||
|
||||
Radixor uses JUnit tags as an explicit execution policy for its test suite.
|
||||
|
||||
The project uses three orthogonal axes:
|
||||
|
||||
1. **Scope** (how the test is executed in the pipeline)
|
||||
2. **Domain** (where in the system it belongs)
|
||||
3. **Intent** (what behavior it verifies)
|
||||
|
||||
## Canonical scope tags
|
||||
|
||||
| Tag | Description | Typical usage |
|
||||
| --- | --- | --- |
|
||||
| `unit` | Fast, deterministic tests that exercise a specific class or behavior without external processes. | Default developer feedback; should stay near-zero flakiness and low run time. |
|
||||
| `integration` | Tests that span multiple components or end-to-end flows of the public pipeline. | Parser/loader/CLI/IO integration checks and multi-step compile-then-load validations. |
|
||||
| `property` | Property-based tests with generator-driven coverage for invariants. | Semantics-preserving laws and edge-case exploration beyond curated fixtures. |
|
||||
| `fuzz` | Randomized stress checks with bounded runtime. | Heavier probabilistic verification of robustness and reduction invariants. |
|
||||
| `compat` | Backward/forward compatibility and reproducibility checks for persisted artifacts. | Artifact fingerprints, deterministic rebuild, and regression fixtures. |
|
||||
| `slow` | Long-running or expensive tests that should not execute in every fast gate. | Heavy fuzz/property budgets or high-duration integration checks. |
|
||||
|
||||
## Canonical domain tags
|
||||
|
||||
| Tag | Description | Typical usage |
|
||||
| --- | --- | --- |
|
||||
| `core` | Core algorithm and foundational platform behavior. | Traversal direction, base data structures, low-level helpers. |
|
||||
| `trie` | All mutable/compiled trie behaviors and traversal internals. | Lookup path selection, node shape, child representation, subtree behavior. |
|
||||
| `frequency-trie` | Algorithms and corner cases specific to frequency-aware trie logic. | Ranking, weighted reductions, persistence of weighted nodes. |
|
||||
| `stemmer` | End-user stemming pipeline semantics. | Parse-encode-apply flows and output invariants. |
|
||||
| `patch` | Patch encoding, decoding, and application semantics. | `PatchCommandEncoder` behavior and related compatibility contracts. |
|
||||
| `io` | Input/output and resource loading boundaries. | Filesystem readers, streams, and stream lifecycle handling. |
|
||||
| `serialization` | Binary persistence contract of compiled artifacts. | Versioned format reads/writes and checksum/consistency checks. |
|
||||
| `parser` | Dictionary and metadata parsing concerns. | Dictionary input parsing and malformed-source rejection. |
|
||||
| `cli` | Command-line entrypoint and command orchestration behavior. | Compile CLI integration and CLI argument validation. |
|
||||
| `metadata` | Trie metadata semantics, compatibility fields, and schema expectations. | Version flags, structural properties, and metadata round-trips. |
|
||||
| `compile` | Compile-time pipeline and build-oriented behavior. | Building, reduction-mode behavior, and compiled artifact generation. |
|
||||
| `diacritic` | Unicode diacritic normalization and stripping behavior. | Accent-removal correctness and locale-safe normalization checks. |
|
||||
|
||||
## Canonical intent tags
|
||||
|
||||
| Tag | Description | Typical usage |
|
||||
| --- | --- | --- |
|
||||
| `construction` | Tests around construction and assembly of runtime structures. | Builders, loaders, and compile-time object construction contracts. |
|
||||
| `lookup` | Read behavior and retrieval semantics. | `get()`, `getAll()`, traversal and missing-key behavior. |
|
||||
| `persistence` | Storage lifecycle semantics. | Serialization/deserialization and round-trip correctness. |
|
||||
| `reduction` | Reduction algorithm correctness and corner cases. | Dominance threshold, subtree deduplication, rank-preservation invariants. |
|
||||
| `encoding` | Encoding transformation direction. | `PatchCommandEncoder.encode` and serialized command form generation. |
|
||||
| `decoding` | Decoding/interpretation of persisted or runtime commands. | Optional consumers that parse and apply encoded command payloads. |
|
||||
| `apply` | Patch application and transformation behavior. | Verifies that applied patches produce expected derived forms. |
|
||||
| `normalization` | Canonicalization and cleanup behavior. | String normalization around case/shape and mirrored input paths. |
|
||||
| `validation` | Input rejection and defensive checks. | Null/empty/invalid contracts and explicit failure conditions. |
|
||||
| `regression` | Guard tests for behavior changes over time. | Known historical bugs and behavioral drift prevention. |
|
||||
| `determinism` | Repeatable results under fixed input and settings. | Compile determinism, stable ordering, and artifact reproducibility. |
|
||||
| `error-handling` | Exception surface and robustness expectations. | Recovery/failure modes and diagnostics quality. |
|
||||
|
||||
## Class-level rules
|
||||
|
||||
1. Every test class has **exactly one** scope tag.
|
||||
2. Every test class has at least one domain tag.
|
||||
3. Additional tags describe intent and may be used on classes or nested tests.
|
||||
4. For each test class, intent tags should reflect the primary behavior under test, not historical naming conventions.
|
||||
|
||||
## Governance and execution policy
|
||||
|
||||
The following rules are used to keep the suite auditable and stable:
|
||||
|
||||
| Rule | Required state | Why |
|
||||
| --- | --- | --- |
|
||||
| Scope discipline | Exactly one scope tag per class. | Prevents accidental promotion of integration-only behavior into fast unit runs. |
|
||||
| Coverage breadth | At least one domain tag per class. | Ensures tests can be grouped by subsystem for targeted review. |
|
||||
| Intent specificity | Use at least one intent tag when behavior is non-trivial. | Makes failure triage faster and profile composition explicit. |
|
||||
| Runtime policy | Never run `slow` tests in the default `unit` profile unless explicitly required. | Preserves turnaround for PR feedback while preserving deep checks. |
|
||||
| Change risk | Any persistence or compatibility-affecting change must include `compat` in validation. | Protects long-lived binary artifact contracts. |
|
||||
| Mutation resistance | `fuzz`/`property` sets should be gated to dedicated profiles. | Limits flakiness exposure and controls CI resource cost. |
|
||||
|
||||
## Suggested CI profiles
|
||||
|
||||
These are recommended launch profiles for local and CI usage and are also exposed as Gradle tasks:
|
||||
|
||||
- **Profile: `ci-smoke` (fast feedback):**
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=unit -DexcludeTags=slow
|
||||
./gradlew ciSmoke
|
||||
```
|
||||
|
||||
`ciSmoke` also excludes `org.egothor.stemmer.CompileIntegrationTest*` at test-name filter level as a
|
||||
defensive fallback in case of future tag drift.
|
||||
`ciRelease` also excludes
|
||||
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` at filter level.
|
||||
|
||||
- **Profile: `ci-core` (core behavioral coverage):**
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=unit,trie,frequency-trie,property
|
||||
./gradlew ciCore
|
||||
```
|
||||
|
||||
- **Profile: `ci-integration` (pipeline correctness):**
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=integration
|
||||
./gradlew ciIntegration
|
||||
```
|
||||
|
||||
- **Profile: `ci-slow` (explicit heavy validation):**
|
||||
|
||||
```
|
||||
./gradlew ciSlow
|
||||
```
|
||||
|
||||
- **Profile: `ci-compat` (artifact stability):**
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=compat,regression
|
||||
./gradlew ciCompat
|
||||
```
|
||||
|
||||
- **Profile: `ci-release` (strong confidence before release):**
|
||||
|
||||
```
|
||||
./gradlew test -DexcludeTags=slow
|
||||
./gradlew ciRelease
|
||||
```
|
||||
`ciRelease` is non-slow by policy and uses the same defensive name-based exclusion for
|
||||
`org.egothor.stemmer.CompileIntegrationTest*` and
|
||||
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` in addition to tag filtering.
|
||||
|
||||
- **Profile: `ci-nightly` (extended hardening):**
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=fuzz
|
||||
./gradlew ciNightly
|
||||
```
|
||||
|
||||
- **Profile: `ci` (enterprise umbrella):**
|
||||
|
||||
```
|
||||
./gradlew ci
|
||||
```
|
||||
|
||||
`ci` and `ciRelease` intentionally do **not** include `slow` paths. Run `ciSlow` explicitly for production-dictionary stress and long-running corpus checks.
|
||||
|
||||
## Practical examples
|
||||
|
||||
All examples use Gradle with JUnit Platform integration:
|
||||
|
||||
- Only unit tests:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=unit
|
||||
```
|
||||
|
||||
- Integration tests only:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=integration
|
||||
```
|
||||
|
||||
- Only trie subsystem tests:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=trie
|
||||
```
|
||||
|
||||
- Deterministic fuzz checks:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=fuzz
|
||||
```
|
||||
|
||||
- Property tests:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=property
|
||||
```
|
||||
|
||||
- Stemmer + patch command behavior:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=stemmer,patch
|
||||
```
|
||||
|
||||
- Compatibility artifacts and regression checks:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=compat
|
||||
```
|
||||
|
||||
- Keep regression suite and remove long-running cases:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=regression -DexcludeTags=slow
|
||||
```
|
||||
|
||||
- Core + patch behavior:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=trie,patch
|
||||
```
|
||||
|
||||
- Deterministic compatibility and persistence checks:
|
||||
|
||||
```
|
||||
./gradlew test -DincludeTags=compat,determinism,serialization
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- `-DincludeTags` and `-DexcludeTags` are interpreted by Gradle task filtering and forwarded into
|
||||
JUnit tag filtering.
|
||||
- Class-name filtering is also available via Gradle test selectors where needed
|
||||
(for example, `--tests *CompileTest`), but tag filtering remains the default
|
||||
execution strategy.
|
||||
- `-DincludeTags` supports comma-separated literal tags. When you need a single exact tag with special
|
||||
characters, quote the argument for the shell.
|
||||
@@ -84,7 +84,7 @@ publishing {
|
||||
}
|
||||
|
||||
signing {
|
||||
required { !version.toString().endsWith('-SNAPSHOT') }
|
||||
required = !version.toString().endsWith('-SNAPSHOT')
|
||||
if (signingKey != null && !signingKey.isBlank()) {
|
||||
useInMemoryPgpKeys(signingKey, signingPassword)
|
||||
sign publishing.publications.mavenJava
|
||||
|
||||
@@ -54,6 +54,7 @@ nav:
|
||||
- Overview: architecture-and-reduction.md
|
||||
- Architecture: architecture.md
|
||||
- Reduction Semantics: reduction-semantics.md
|
||||
- Lookup Edge Optimization: lookup-edge-optimization.md
|
||||
- Compatibility and Guarantees: compatibility-and-guarantees.md
|
||||
|
||||
- Dictionaries:
|
||||
@@ -63,3 +64,4 @@ nav:
|
||||
- Quality and Operations: quality-and-operations.md
|
||||
- Benchmarking: benchmarking.md
|
||||
- Reports: reports.md
|
||||
- Test taxonomy and execution filtering: test-taxonomy-and-filtering.md
|
||||
|
||||
@@ -31,11 +31,13 @@
|
||||
package org.egothor.stemmer.benchmark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.egothor.stemmer.FrequencyTrie;
|
||||
import org.egothor.stemmer.PatchCommandEncoder;
|
||||
import org.egothor.stemmer.ReductionMode;
|
||||
import org.egothor.stemmer.ReductionSettings;
|
||||
import org.egothor.stemmer.ValueCount;
|
||||
import org.openjdk.jmh.annotations.Benchmark;
|
||||
import org.openjdk.jmh.annotations.BenchmarkMode;
|
||||
import org.openjdk.jmh.annotations.Level;
|
||||
@@ -97,12 +99,45 @@ public class FrequencyTrieLookupBenchmark {
|
||||
*/
|
||||
private String[] lookupKeys;
|
||||
|
||||
/**
|
||||
* Lookup keys as normalized caller-owned character storage.
|
||||
*/
|
||||
private char[][] lookupKeyCharacters;
|
||||
|
||||
/**
|
||||
* Keys that are known to return multiple patch candidates from
|
||||
* {@code getAll()}.
|
||||
*/
|
||||
private String[] ambiguousLookupKeys;
|
||||
|
||||
/**
|
||||
* Ambiguous lookup keys as normalized caller-owned character storage.
|
||||
*/
|
||||
private char[][] ambiguousLookupKeyCharacters;
|
||||
|
||||
/**
|
||||
* Preferred patches aligned with {@link #lookupKeys}.
|
||||
*/
|
||||
private String[] preferredPatches;
|
||||
|
||||
/**
|
||||
* Reusable output buffer for patch application benchmarks.
|
||||
*/
|
||||
private char[] outputBuffer;
|
||||
|
||||
/**
|
||||
* Mutable field consumed by visitor sinks.
|
||||
*/
|
||||
private int visitorAccumulator;
|
||||
|
||||
/**
|
||||
* Sink used by visitor lookup benchmarks without per-invocation allocation.
|
||||
*/
|
||||
private final FrequencyTrie.EntrySink<String> visitorSink = (value, count, rank) -> {
|
||||
this.visitorAccumulator += value.length() + count + rank;
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Initializes the benchmark state.
|
||||
*
|
||||
@@ -116,6 +151,23 @@ public class FrequencyTrieLookupBenchmark {
|
||||
this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true);
|
||||
this.lookupKeys = corpus.lookupKeys();
|
||||
this.ambiguousLookupKeys = corpus.ambiguousLookupKeys();
|
||||
this.lookupKeyCharacters = toCharArrays(this.lookupKeys);
|
||||
this.ambiguousLookupKeyCharacters = toCharArrays(this.ambiguousLookupKeys);
|
||||
this.preferredPatches = new String[this.lookupKeys.length];
|
||||
int maxKeyLength = 0;
|
||||
for (int index = 0; index < this.lookupKeys.length; index++) {
|
||||
this.preferredPatches[index] = this.trie.get(this.lookupKeys[index]);
|
||||
maxKeyLength = Math.max(maxKeyLength, this.lookupKeys[index].length());
|
||||
}
|
||||
this.outputBuffer = new char[maxKeyLength + 32];
|
||||
}
|
||||
|
||||
private static char[][] toCharArrays(final String[] values) {
|
||||
final char[][] characters = new char[values.length][];
|
||||
for (int index = 0; index < values.length; index++) {
|
||||
characters[index] = values[index].toCharArray();
|
||||
}
|
||||
return characters;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,6 +207,61 @@ public class FrequencyTrieLookupBenchmark {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures retrieval of all patch candidates through caller-owned normalized
|
||||
* character storage and a visitor sink.
|
||||
*
|
||||
* @param state prepared lookup state
|
||||
* @param blackhole sink preventing dead-code elimination
|
||||
*/
|
||||
@Benchmark
|
||||
public void lookupAllPatchesWithNormalizedCharVisitor(final LookupState state, final Blackhole blackhole) {
|
||||
final char[][] keys = state.ambiguousLookupKeyCharacters;
|
||||
for (char[] key : keys) {
|
||||
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
|
||||
if (count < 2) {
|
||||
throw new IllegalStateException("Expected multiple patches for benchmark key.");
|
||||
}
|
||||
}
|
||||
blackhole.consume(state.visitorAccumulator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures counted candidate retrieval through the allocating entry API.
|
||||
*
|
||||
* @param state prepared lookup state
|
||||
* @param blackhole sink preventing dead-code elimination
|
||||
*/
|
||||
@Benchmark
|
||||
public void lookupPatchEntries(final LookupState state, final Blackhole blackhole) {
|
||||
final String[] keys = state.ambiguousLookupKeys;
|
||||
for (String key : keys) {
|
||||
final List<ValueCount<String>> entries = state.trie.getEntries(key);
|
||||
if (entries.size() < 2) {
|
||||
throw new IllegalStateException("Expected multiple entries for key " + key + '.');
|
||||
}
|
||||
blackhole.consume(entries);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures counted candidate retrieval through the visitor API.
|
||||
*
|
||||
* @param state prepared lookup state
|
||||
* @param blackhole sink preventing dead-code elimination
|
||||
*/
|
||||
@Benchmark
|
||||
public void lookupPatchEntriesWithVisitor(final LookupState state, final Blackhole blackhole) {
|
||||
final char[][] keys = state.ambiguousLookupKeyCharacters;
|
||||
for (char[] key : keys) {
|
||||
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
|
||||
if (count < 2) {
|
||||
throw new IllegalStateException("Expected multiple entries for benchmark key.");
|
||||
}
|
||||
}
|
||||
blackhole.consume(state.visitorAccumulator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures end-to-end preferred stemming from lookup plus patch application.
|
||||
*
|
||||
@@ -170,6 +277,28 @@ public class FrequencyTrieLookupBenchmark {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures patch application into caller-owned output storage.
|
||||
*
|
||||
* @param state prepared lookup state
|
||||
* @param blackhole sink preventing dead-code elimination
|
||||
*/
|
||||
@Benchmark
|
||||
public void applyPreferredPatchToBuffer(final LookupState state, final Blackhole blackhole) {
|
||||
final String[] keys = state.lookupKeys;
|
||||
final String[] patches = state.preferredPatches;
|
||||
final char[] output = state.outputBuffer;
|
||||
for (int index = 0; index < keys.length; index++) {
|
||||
final int length = PatchCommandEncoder.applyTo(keys[index], patches[index],
|
||||
state.trie.traversalDirection(), output, 0, output.length);
|
||||
if (length == PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
|
||||
throw new IllegalStateException("Output buffer too small for key " + keys[index] + '.');
|
||||
}
|
||||
blackhole.consume(length);
|
||||
blackhole.consume(output[0]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures end-to-end full candidate stemming from {@code getAll()} plus
|
||||
* patch application.
|
||||
|
||||
5
src/main/java/module-info.java
Normal file
5
src/main/java/module-info.java
Normal file
@@ -0,0 +1,5 @@
|
||||
module org.egothor.radixor {
|
||||
requires java.logging;
|
||||
|
||||
exports org.egothor.stemmer;
|
||||
}
|
||||
@@ -48,8 +48,7 @@ public enum CaseProcessingMode {
|
||||
AS_IS,
|
||||
|
||||
/**
|
||||
* Normalizes all dictionary content to lower case using
|
||||
* {@link Locale#ROOT}.
|
||||
* Normalizes all dictionary content to lower case using {@link Locale#ROOT}.
|
||||
*/
|
||||
LOWERCASE_WITH_LOCALE_ROOT
|
||||
}
|
||||
|
||||
@@ -93,12 +93,12 @@ final class DiacriticStripper {
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes supported diacritic marks and common Latin ligatures from the supplied
|
||||
* text.
|
||||
* Removes supported diacritic marks and common Latin ligatures from the
|
||||
* supplied text.
|
||||
*
|
||||
* <p>
|
||||
* The method returns the original {@link String} instance when no replacement is
|
||||
* required, avoiding an unnecessary allocation on the common ASCII path.
|
||||
* The method returns the original {@link String} instance when no replacement
|
||||
* is required, avoiding an unnecessary allocation on the common ASCII path.
|
||||
* </p>
|
||||
*
|
||||
* @param input text to normalize
|
||||
|
||||
@@ -51,7 +51,6 @@ import java.util.logging.Logger;
|
||||
import org.egothor.stemmer.trie.CompiledNode;
|
||||
import org.egothor.stemmer.trie.LocalValueSummary;
|
||||
import org.egothor.stemmer.trie.MutableNode;
|
||||
import org.egothor.stemmer.trie.NodeData;
|
||||
import org.egothor.stemmer.trie.ReducedNode;
|
||||
import org.egothor.stemmer.trie.ReductionContext;
|
||||
import org.egothor.stemmer.trie.ReductionSignature;
|
||||
@@ -87,7 +86,6 @@ import org.egothor.stemmer.trie.ReductionSignature;
|
||||
*
|
||||
* @param <V> value type
|
||||
*/
|
||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||
public final class FrequencyTrie<V> {
|
||||
|
||||
/**
|
||||
@@ -121,7 +119,8 @@ public final class FrequencyTrie<V> {
|
||||
private final boolean removeDiacritics;
|
||||
|
||||
/**
|
||||
* Shared empty array instance for empty lookup results from {@link #getAll(String)}.
|
||||
* Shared empty array instance for empty lookup results from
|
||||
* {@link #getAll(String)}.
|
||||
*/
|
||||
private final V[] emptyValues;
|
||||
|
||||
@@ -130,11 +129,59 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private static final int STREAM_MAGIC = 0x45475452;
|
||||
|
||||
/**
|
||||
* Minimum supported stream version constant retained for explicit range checks.
|
||||
*/
|
||||
private static final int MIN_STREAM_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Number of stored values for which {@link #getEntries(String)} can return an
|
||||
* empty result.
|
||||
*/
|
||||
private static final int NO_VALUE_COUNT = 0;
|
||||
|
||||
/**
|
||||
* Number of stored values for which {@link #getEntries(String)} can use a
|
||||
* one-item immutable list special case.
|
||||
*/
|
||||
private static final int SINGLE_VALUE_COUNT = 1;
|
||||
|
||||
/**
|
||||
* Binary format version.
|
||||
*/
|
||||
private static final int STREAM_VERSION = 5;
|
||||
|
||||
/**
|
||||
* Version where traversal-direction ordinal is persisted.
|
||||
*/
|
||||
private static final int TRAVERSAL_VERSION = 2;
|
||||
|
||||
/**
|
||||
* Version where compact reduction metadata is persisted.
|
||||
*/
|
||||
private static final int REDUCTION_VERSION = 3;
|
||||
|
||||
/**
|
||||
* Version where case-processing mode ordinal is persisted.
|
||||
*/
|
||||
private static final int CASE_VERSION = 4;
|
||||
|
||||
/**
|
||||
* Argument name for lookup keys.
|
||||
*/
|
||||
private static final String ARG_KEY = "key";
|
||||
|
||||
/**
|
||||
* Default dense child lookup span in code points used when materializing
|
||||
* compiled nodes without an explicit override.
|
||||
* <p>
|
||||
* Increasing this value increases the chance of direct array indexing for child
|
||||
* lookup at runtime at the cost of per-node dense table memory for compact
|
||||
* character spans.
|
||||
* </p>
|
||||
*/
|
||||
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
|
||||
|
||||
/**
|
||||
* Returns the current persisted binary stream format version.
|
||||
*
|
||||
@@ -150,6 +197,30 @@ public final class FrequencyTrie<V> {
|
||||
return STREAM_VERSION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives trie values during visitor-style lookup.
|
||||
*
|
||||
* <p>
|
||||
* Implementations are caller-owned and are not retained by the trie. Returning
|
||||
* {@code false} stops iteration after the current callback.
|
||||
* </p>
|
||||
*
|
||||
* @param <V> value type
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface EntrySink<V> {
|
||||
|
||||
/**
|
||||
* Accepts one ordered local value.
|
||||
*
|
||||
* @param value stored value
|
||||
* @param count stored local occurrence count
|
||||
* @param rank zero-based rank in deterministic local ordering
|
||||
* @return {@code true} to continue iteration, {@code false} to stop
|
||||
*/
|
||||
boolean accept(V value, int count, int rank);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new compiled trie instance.
|
||||
*
|
||||
@@ -188,7 +259,7 @@ public final class FrequencyTrie<V> {
|
||||
* @throws NullPointerException if {@code key} is {@code null}
|
||||
*/
|
||||
public V get(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null) {
|
||||
return null;
|
||||
@@ -225,7 +296,7 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public V[] getAll(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null) {
|
||||
return this.emptyValues;
|
||||
@@ -259,9 +330,8 @@ public final class FrequencyTrie<V> {
|
||||
* if the key does not exist or no value is stored at the addressed node
|
||||
* @throws NullPointerException if {@code key} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
public List<ValueCount<V>> getEntries(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null) {
|
||||
return List.of();
|
||||
@@ -269,11 +339,11 @@ public final class FrequencyTrie<V> {
|
||||
|
||||
final V[] orderedValues = node.orderedValues();
|
||||
final int valueCount = orderedValues.length;
|
||||
if (valueCount == 0) {
|
||||
if (valueCount == NO_VALUE_COUNT) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
if (valueCount == 1) {
|
||||
if (valueCount == SINGLE_VALUE_COUNT) {
|
||||
return List.of(new ValueCount<>(orderedValues[0], node.orderedCounts()[0]));
|
||||
}
|
||||
|
||||
@@ -285,6 +355,132 @@ public final class FrequencyTrie<V> {
|
||||
return Collections.unmodifiableList(entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits all values stored at the node addressed by an already-normalized
|
||||
* {@code char[]} key slice.
|
||||
*
|
||||
* <p>
|
||||
* This method bypasses {@link TrieMetadata#caseProcessingMode()} and
|
||||
* {@link TrieMetadata#diacriticProcessingMode()}. The caller must provide input
|
||||
* normalized exactly as required by this trie's metadata. The trie is immutable
|
||||
* and thread-safe for concurrent reads; the supplied sink is caller-owned and
|
||||
* is not retained.
|
||||
* </p>
|
||||
*
|
||||
* @param key normalized key storage
|
||||
* @param offset first character offset
|
||||
* @param length number of characters to read
|
||||
* @param sink value sink
|
||||
* @param maxResults maximum number of results to visit
|
||||
* @return number of visited values
|
||||
* @throws NullPointerException if {@code key} or {@code sink} is
|
||||
* {@code null}
|
||||
* @throws IndexOutOfBoundsException if the key slice is invalid
|
||||
* @throws IllegalArgumentException if {@code maxResults} is negative
|
||||
*/
|
||||
public int getAllNormalized(final char[] key, final int offset, final int length, final EntrySink<? super V> sink,
|
||||
final int maxResults) {
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
Objects.requireNonNull(sink, "sink");
|
||||
Objects.checkFromIndexSize(offset, length, key.length);
|
||||
validateMaxResults(maxResults);
|
||||
if (maxResults == 0) {
|
||||
return 0;
|
||||
}
|
||||
return visitNode(findNode(key, offset, length), sink, maxResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits all values stored at the node addressed by an already-normalized
|
||||
* character sequence.
|
||||
*
|
||||
* @param key normalized key
|
||||
* @param sink value sink
|
||||
* @param maxResults maximum number of results to visit
|
||||
* @return number of visited values
|
||||
* @throws NullPointerException if {@code key} or {@code sink} is
|
||||
* {@code null}
|
||||
* @throws IllegalArgumentException if {@code maxResults} is negative
|
||||
* @see #getAllNormalized(char[], int, int, EntrySink, int)
|
||||
*/
|
||||
public int getAllNormalized(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
Objects.requireNonNull(sink, "sink");
|
||||
validateMaxResults(maxResults);
|
||||
if (maxResults == 0) {
|
||||
return 0;
|
||||
}
|
||||
return visitNode(findNode(key), sink, maxResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits the first value stored at the node addressed by an already-normalized
|
||||
* {@code char[]} key slice.
|
||||
*
|
||||
* @param key normalized key storage
|
||||
* @param offset first character offset
|
||||
* @param length number of characters to read
|
||||
* @param sink value sink
|
||||
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||
* @see #getAllNormalized(char[], int, int, EntrySink, int)
|
||||
*/
|
||||
public boolean getFirstNormalized(final char[] key, final int offset, final int length,
|
||||
final EntrySink<? super V> sink) {
|
||||
return getAllNormalized(key, offset, length, sink, 1) == 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits the first value stored at the node addressed by an already-normalized
|
||||
* character sequence.
|
||||
*
|
||||
* @param key normalized key
|
||||
* @param sink value sink
|
||||
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||
* @see #getAllNormalized(CharSequence, EntrySink, int)
|
||||
*/
|
||||
public boolean getFirstNormalized(final CharSequence key, final EntrySink<? super V> sink) {
|
||||
return getAllNormalized(key, sink, 1) == 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits all values stored at the node addressed by the supplied key, applying
|
||||
* metadata-driven lookup normalization when required.
|
||||
*
|
||||
* <p>
|
||||
* This method preserves the same lookup normalization semantics as
|
||||
* {@link #getAll(String)}. It may allocate when metadata requires lowercase or
|
||||
* diacritic normalization.
|
||||
* </p>
|
||||
*
|
||||
* @param key key to resolve
|
||||
* @param sink value sink
|
||||
* @param maxResults maximum number of results to visit
|
||||
* @return number of visited values
|
||||
*/
|
||||
public int getAll(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
Objects.requireNonNull(sink, "sink");
|
||||
validateMaxResults(maxResults);
|
||||
if (maxResults == 0) {
|
||||
return 0;
|
||||
}
|
||||
final CharSequence normalized = normalizeLookupKey(key);
|
||||
return visitNode(findNode(normalized), sink, maxResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits the first value stored at the node addressed by the supplied key,
|
||||
* applying metadata-driven lookup normalization when required.
|
||||
*
|
||||
* @param key key to resolve
|
||||
* @param sink value sink
|
||||
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||
* @see #getAll(CharSequence, EntrySink, int)
|
||||
*/
|
||||
public boolean getFirst(final CharSequence key, final EntrySink<? super V> sink) {
|
||||
return getAll(key, sink, 1) == 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the logical key traversal direction used by this trie.
|
||||
*
|
||||
@@ -383,47 +579,32 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||
final ValueStreamCodec<V> valueCodec) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
Objects.requireNonNull(valueCodec, "valueCodec");
|
||||
return readFrom(inputStream, arrayFactory, valueCodec, -1);
|
||||
}
|
||||
|
||||
final DataInputStream dataInput; // NOPMD
|
||||
if (inputStream instanceof DataInputStream) {
|
||||
dataInput = (DataInputStream) inputStream;
|
||||
} else {
|
||||
dataInput = new DataInputStream(inputStream);
|
||||
}
|
||||
|
||||
final int magic = dataInput.readInt();
|
||||
if (magic != STREAM_MAGIC) {
|
||||
throw new IOException("Unsupported trie stream header: " + Integer.toHexString(magic));
|
||||
}
|
||||
|
||||
final int version = dataInput.readInt();
|
||||
if (version != 1 && version != 3 && version != 4 && version != STREAM_VERSION) {
|
||||
throw new IOException("Unsupported trie stream version: " + version);
|
||||
}
|
||||
|
||||
final int nodeCount = dataInput.readInt();
|
||||
if (nodeCount < 0) {
|
||||
throw new IOException("Negative node count: " + nodeCount);
|
||||
}
|
||||
|
||||
final int rootNodeId = dataInput.readInt();
|
||||
if (rootNodeId < 0 || rootNodeId >= nodeCount) {
|
||||
throw new IOException("Invalid root node id: " + rootNodeId);
|
||||
}
|
||||
|
||||
final TrieMetadata metadata = readMetadata(dataInput, version);
|
||||
|
||||
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount);
|
||||
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
||||
|
||||
if (LOGGER.isLoggable(Level.FINE)) {
|
||||
LOGGER.log(Level.FINE, "Read compiled trie with {0} canonical nodes.", nodeCount);
|
||||
}
|
||||
|
||||
return new FrequencyTrie<>(arrayFactory, rootNode, metadata);
|
||||
/**
|
||||
* Reads a compiled trie from the supplied input stream, optionally overriding
|
||||
* dense child-index span configuration.
|
||||
* <p>
|
||||
* This setting is applied only while materializing the in-memory compiled
|
||||
* representation during load. It is not serialized in {@link TrieMetadata}, so
|
||||
* each load can independently choose its own runtime lookup trade-off.
|
||||
* </p>
|
||||
*
|
||||
* @param inputStream source input stream
|
||||
* @param arrayFactory array factory used to create typed arrays
|
||||
* @param valueCodec codec used to read values
|
||||
* @param maxExpandedIndex dense lookup span override; zero disables dense
|
||||
* lookup, negative values use
|
||||
* {@link #DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @param <V> value type
|
||||
* @return deserialized compiled trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if reading fails or the binary format is invalid
|
||||
*/
|
||||
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
||||
return CompiledTrieReader.read(inputStream, arrayFactory, valueCodec, maxExpandedIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -438,73 +619,6 @@ public final class FrequencyTrie<V> {
|
||||
dataOutput.writeUTF(metadata.toTextBlock());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads persisted trie metadata while remaining backward compatible with
|
||||
* earlier stream versions.
|
||||
*
|
||||
* @param dataInput input stream
|
||||
* @param version persisted stream version
|
||||
* @return deserialized metadata
|
||||
* @throws IOException if the metadata section is invalid
|
||||
*/
|
||||
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version) throws IOException {
|
||||
if (version >= 5) { // NOPMD
|
||||
try {
|
||||
return TrieMetadata.fromTextBlock(version, dataInput.readUTF());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IOException("Invalid metadata block.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
final WordTraversalDirection traversalDirection;
|
||||
if (version >= 2) { // NOPMD
|
||||
final int traversalDirectionOrdinal = dataInput.readInt();
|
||||
final WordTraversalDirection[] traversalDirections = WordTraversalDirection.values();
|
||||
if (traversalDirectionOrdinal < 0 || traversalDirectionOrdinal >= traversalDirections.length) {
|
||||
throw new IOException("Invalid traversal direction ordinal: " + traversalDirectionOrdinal);
|
||||
}
|
||||
traversalDirection = traversalDirections[traversalDirectionOrdinal];
|
||||
} else {
|
||||
traversalDirection = WordTraversalDirection.BACKWARD;
|
||||
}
|
||||
|
||||
if (version < 3) { // NOPMD
|
||||
return TrieMetadata.legacy(version, traversalDirection);
|
||||
}
|
||||
|
||||
final ReductionMode[] reductionModes = ReductionMode.values();
|
||||
final int reductionModeOrdinal = dataInput.readInt();
|
||||
if (reductionModeOrdinal < 0 || reductionModeOrdinal >= reductionModes.length) {
|
||||
throw new IOException("Invalid reduction mode ordinal: " + reductionModeOrdinal);
|
||||
}
|
||||
|
||||
final int dominantWinnerMinPercent = dataInput.readInt();
|
||||
final int dominantWinnerOverSecondRatio = dataInput.readInt(); // NOPMD
|
||||
|
||||
final DiacriticProcessingMode[] diacriticProcessingModes = DiacriticProcessingMode.values();
|
||||
final int diacriticProcessingModeOrdinal = dataInput.readInt(); // NOPMD
|
||||
if (diacriticProcessingModeOrdinal < 0 || diacriticProcessingModeOrdinal >= diacriticProcessingModes.length) {
|
||||
throw new IOException("Invalid diacritic processing mode ordinal: " + diacriticProcessingModeOrdinal);
|
||||
}
|
||||
|
||||
final CaseProcessingMode caseProcessingMode;
|
||||
if (version >= 4) { // NOPMD
|
||||
final CaseProcessingMode[] caseProcessingModes = CaseProcessingMode.values();
|
||||
final int caseProcessingModeOrdinal = dataInput.readInt();
|
||||
if (caseProcessingModeOrdinal < 0 || caseProcessingModeOrdinal >= caseProcessingModes.length) {
|
||||
throw new IOException("Invalid case processing mode ordinal: " + caseProcessingModeOrdinal);
|
||||
}
|
||||
caseProcessingMode = caseProcessingModes[caseProcessingModeOrdinal];
|
||||
} else {
|
||||
caseProcessingMode = CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||
}
|
||||
|
||||
return new TrieMetadata(version, traversalDirection,
|
||||
new ReductionSettings(reductionModes[reductionModeOrdinal], dominantWinnerMinPercent,
|
||||
dominantWinnerOverSecondRatio),
|
||||
diacriticProcessingModes[diacriticProcessingModeOrdinal], caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of canonical compiled nodes reachable from the root.
|
||||
*
|
||||
@@ -574,103 +688,222 @@ public final class FrequencyTrie<V> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all compiled nodes and resolves child references.
|
||||
*
|
||||
* @param dataInput input
|
||||
* @param arrayFactory array factory
|
||||
* @param valueCodec value codec
|
||||
* @param nodeCount number of nodes
|
||||
* @param <V> value type
|
||||
* @return array of nodes indexed by serialized node identifier
|
||||
* @throws IOException if reading fails or the stream is invalid
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
|
||||
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
|
||||
final ValueStreamCodec<V> valueCodec, final int nodeCount) throws IOException {
|
||||
final List<NodeData<V>> nodeDataList = new ArrayList<>(nodeCount);
|
||||
|
||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||
final int edgeCount = dataInput.readInt();
|
||||
if (edgeCount < 0) {
|
||||
throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount);
|
||||
}
|
||||
|
||||
final char[] edgeLabels = new char[edgeCount];
|
||||
final int[] childNodeIds = new int[edgeCount];
|
||||
|
||||
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
||||
edgeLabels[edgeIndex] = dataInput.readChar();
|
||||
childNodeIds[edgeIndex] = dataInput.readInt();
|
||||
}
|
||||
|
||||
validateSerializedEdges(nodeIndex, edgeLabels);
|
||||
|
||||
final int valueCount = dataInput.readInt();
|
||||
if (valueCount < 0) {
|
||||
throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount);
|
||||
}
|
||||
|
||||
final V[] orderedValues = arrayFactory.apply(valueCount);
|
||||
final int[] orderedCounts = new int[valueCount];
|
||||
|
||||
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
|
||||
orderedValues[valueIndex] = valueCodec.read(dataInput);
|
||||
orderedCounts[valueIndex] = dataInput.readInt();
|
||||
if (orderedCounts[valueIndex] <= 0) {
|
||||
throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index "
|
||||
+ valueIndex + ": " + orderedCounts[valueIndex]);
|
||||
}
|
||||
}
|
||||
|
||||
nodeDataList.add(new NodeData<>(edgeLabels, childNodeIds, orderedValues, orderedCounts));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<V>[] nodes = new CompiledNode[nodeCount];
|
||||
|
||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||
final NodeData<V> nodeData = nodeDataList.get(nodeIndex);
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<V>[] children = new CompiledNode[nodeData.childNodeIds().length];
|
||||
nodes[nodeIndex] = new CompiledNode<>(nodeData.edgeLabels(), children, nodeData.orderedValues(),
|
||||
nodeData.orderedCounts());
|
||||
}
|
||||
|
||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||
final NodeData<V> nodeData = nodeDataList.get(nodeIndex);
|
||||
final CompiledNode<V> node = nodes[nodeIndex];
|
||||
|
||||
for (int edgeIndex = 0; edgeIndex < nodeData.childNodeIds().length; edgeIndex++) {
|
||||
final int childNodeId = nodeData.childNodeIds()[edgeIndex];
|
||||
if (childNodeId < 0 || childNodeId >= nodeCount) {
|
||||
throw new IOException("Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex
|
||||
+ ": " + childNodeId);
|
||||
}
|
||||
node.children()[edgeIndex] = nodes[childNodeId];
|
||||
}
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the serialized edge-label sequence for one node.
|
||||
* Internal helper that materializes serialized trie data.
|
||||
*
|
||||
* <p>
|
||||
* Compiled nodes rely on binary search for child lookup and therefore require
|
||||
* edge labels to be stored in strict ascending order without duplicates.
|
||||
* Rejecting malformed streams here keeps lookup semantics deterministic and
|
||||
* avoids silently constructing a trie whose search behavior would be undefined.
|
||||
*
|
||||
* @param nodeIndex serialized node identifier
|
||||
* @param edgeLabels serialized edge labels
|
||||
* @throws IOException if the edge labels are not strictly ascending
|
||||
* Moving reader complexity into this helper keeps the public-facing class from
|
||||
* accumulating excessive class-level cyclomatic complexity while preserving the
|
||||
* same binary compatibility contract.
|
||||
* </p>
|
||||
*/
|
||||
private static void validateSerializedEdges(final int nodeIndex, final char... edgeLabels) throws IOException {
|
||||
for (int edgeIndex = 1; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||
if (edgeLabels[edgeIndex - 1] >= edgeLabels[edgeIndex]) {
|
||||
throw new IOException("Edge labels must be strictly ascending at node " + nodeIndex + ", edge index "
|
||||
+ edgeIndex + ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
|
||||
private static final class CompiledTrieReader {
|
||||
|
||||
private static <V> FrequencyTrie<V> read(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
Objects.requireNonNull(valueCodec, "valueCodec");
|
||||
if (maxExpandedIndex < -1) {
|
||||
throw new IllegalArgumentException("maxExpandedIndex must be >= -1.");
|
||||
}
|
||||
|
||||
final DataInputStream dataInput = wrapInputStream(inputStream);
|
||||
final int magic = dataInput.readInt();
|
||||
if (magic != STREAM_MAGIC) {
|
||||
throw new IOException("Unsupported trie stream header: " + Integer.toHexString(magic));
|
||||
}
|
||||
|
||||
final int version = dataInput.readInt();
|
||||
if (version < MIN_STREAM_VERSION || version > STREAM_VERSION) {
|
||||
throw new IOException("Unsupported trie stream version: " + version);
|
||||
}
|
||||
|
||||
final int nodeCount = dataInput.readInt();
|
||||
if (nodeCount < 0) {
|
||||
throw new IOException("Negative node count: " + nodeCount);
|
||||
}
|
||||
|
||||
final int rootNodeId = dataInput.readInt();
|
||||
if (rootNodeId < 0 || rootNodeId >= nodeCount) {
|
||||
throw new IOException("Invalid root node id: " + rootNodeId);
|
||||
}
|
||||
|
||||
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
|
||||
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
|
||||
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount,
|
||||
effectiveMaxExpandedIndex);
|
||||
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
||||
|
||||
if (LOGGER.isLoggable(Level.FINE)) {
|
||||
LOGGER.log(Level.FINE, "Read compiled trie with {0} canonical nodes.", nodeCount);
|
||||
}
|
||||
|
||||
return new FrequencyTrie<>(arrayFactory, rootNode, sourceMetadata);
|
||||
}
|
||||
|
||||
private static DataInputStream wrapInputStream(final InputStream inputStream) {
|
||||
return inputStream instanceof DataInputStream ? (DataInputStream) inputStream
|
||||
: new DataInputStream(inputStream);
|
||||
}
|
||||
|
||||
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version)
|
||||
throws IOException {
|
||||
if (version == STREAM_VERSION) {
|
||||
return readTextMetadata(dataInput);
|
||||
}
|
||||
|
||||
final WordTraversalDirection traversalDirection = readTraversalDirection(dataInput, version);
|
||||
if (version < REDUCTION_VERSION) {
|
||||
return TrieMetadata.legacy(version, traversalDirection);
|
||||
}
|
||||
|
||||
final ReductionSettings reductionSettings = readReductionSettings(dataInput);
|
||||
final DiacriticProcessingMode diacriticProcessingMode = readEnumByOrdinal(dataInput,
|
||||
DiacriticProcessingMode.values(), "diacritic processing mode");
|
||||
final CaseProcessingMode caseProcessingMode = version >= CASE_VERSION ? readCaseProcessingMode(dataInput)
|
||||
: CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||
return new TrieMetadata(version, traversalDirection, reductionSettings, diacriticProcessingMode,
|
||||
caseProcessingMode);
|
||||
}
|
||||
|
||||
private static TrieMetadata readTextMetadata(final DataInputStream dataInput) throws IOException {
|
||||
try {
|
||||
return TrieMetadata.fromTextBlock(STREAM_VERSION, dataInput.readUTF());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IOException("Invalid metadata block.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static WordTraversalDirection readTraversalDirection(final DataInputStream dataInput, final int version)
|
||||
throws IOException {
|
||||
if (version < TRAVERSAL_VERSION) {
|
||||
return WordTraversalDirection.BACKWARD;
|
||||
}
|
||||
return readEnumByOrdinal(dataInput, WordTraversalDirection.values(), "traversal direction");
|
||||
}
|
||||
|
||||
private static ReductionSettings readReductionSettings(final DataInputStream dataInput) throws IOException {
|
||||
final ReductionMode reductionMode = readEnumByOrdinal(dataInput, ReductionMode.values(), "reduction mode");
|
||||
final int dominantWinnerMinPercent = dataInput.readInt();
|
||||
final int dominantWinnerOverSecondRatio = dataInput.readInt(); // NOPMD
|
||||
return new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio);
|
||||
}
|
||||
|
||||
private static CaseProcessingMode readCaseProcessingMode(final DataInputStream dataInput) throws IOException {
|
||||
return readEnumByOrdinal(dataInput, CaseProcessingMode.values(), "case processing mode");
|
||||
}
|
||||
|
||||
private static <E extends Enum<E>> E readEnumByOrdinal(final DataInputStream dataInput, final E[] values,
|
||||
final String name) throws IOException {
|
||||
final int ordinal = dataInput.readInt();
|
||||
if (ordinal < 0 || ordinal >= values.length) {
|
||||
throw new IOException("Invalid " + name + " ordinal: " + ordinal);
|
||||
}
|
||||
return values[ordinal];
|
||||
}
|
||||
|
||||
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
|
||||
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final int nodeCount,
|
||||
final int maxExpandedIndex) throws IOException {
|
||||
final char[][] edgeLabelsByNode = new char[nodeCount][];
|
||||
final int[][] childNodeIdsByNode = new int[nodeCount][];
|
||||
@SuppressWarnings("unchecked")
|
||||
final V[][] orderedValuesByNode = (V[][]) new Object[nodeCount][];
|
||||
final int[][] orderedCountsByNode = new int[nodeCount][];
|
||||
|
||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||
final int edgeCount = dataInput.readInt();
|
||||
if (edgeCount < 0) {
|
||||
throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount);
|
||||
}
|
||||
|
||||
edgeLabelsByNode[nodeIndex] = new char[edgeCount];
|
||||
childNodeIdsByNode[nodeIndex] = new int[edgeCount];
|
||||
|
||||
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
||||
edgeLabelsByNode[nodeIndex][edgeIndex] = dataInput.readChar();
|
||||
childNodeIdsByNode[nodeIndex][edgeIndex] = dataInput.readInt();
|
||||
}
|
||||
|
||||
validateSerializedEdges(nodeIndex, edgeLabelsByNode[nodeIndex]);
|
||||
|
||||
final int valueCount = dataInput.readInt();
|
||||
if (valueCount < 0) {
|
||||
throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount);
|
||||
}
|
||||
|
||||
orderedValuesByNode[nodeIndex] = arrayFactory.apply(valueCount);
|
||||
orderedCountsByNode[nodeIndex] = new int[valueCount];
|
||||
|
||||
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
|
||||
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
|
||||
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
|
||||
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
|
||||
throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index "
|
||||
+ valueIndex + ": " + orderedCountsByNode[nodeIndex][valueIndex]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<V>[] nodes = new CompiledNode[nodeCount];
|
||||
final boolean[] inProgress = new boolean[nodeCount];
|
||||
|
||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||
nodes[nodeIndex] = resolveNode(nodeIndex, edgeLabelsByNode, childNodeIdsByNode, orderedValuesByNode,
|
||||
orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
private static <V> CompiledNode<V> resolveNode(final int nodeIndex, final char[][] edgeLabelsByNode,
|
||||
final int[][] childNodeIdsByNode, final V[][] orderedValuesByNode, final int[][] orderedCountsByNode,
|
||||
final CompiledNode<V>[] nodes, final boolean[] inProgress, final int maxExpandedIndex)
|
||||
throws IOException {
|
||||
final CompiledNode<V> cachedNode = nodes[nodeIndex];
|
||||
if (cachedNode != null) {
|
||||
return cachedNode;
|
||||
}
|
||||
|
||||
if (inProgress[nodeIndex]) {
|
||||
throw new IOException(
|
||||
"Invalid serialized node graph: cyclic reference detected at node " + nodeIndex + '.');
|
||||
}
|
||||
inProgress[nodeIndex] = true;
|
||||
try {
|
||||
final char[] edgeLabels = edgeLabelsByNode[nodeIndex];
|
||||
final int[] childNodeIds = childNodeIdsByNode[nodeIndex];
|
||||
final int edgeCount = childNodeIds.length;
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<V>[] children = new CompiledNode[edgeCount];
|
||||
|
||||
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
||||
final int childNodeId = childNodeIds[edgeIndex];
|
||||
if (childNodeId < 0 || childNodeId >= edgeLabelsByNode.length) {
|
||||
throw new IOException("Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex
|
||||
+ ": " + childNodeId);
|
||||
}
|
||||
children[edgeIndex] = resolveNode(childNodeId, edgeLabelsByNode, childNodeIdsByNode,
|
||||
orderedValuesByNode, orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
|
||||
}
|
||||
|
||||
final CompiledNode<V> node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex],
|
||||
maxExpandedIndex, orderedCountsByNode[nodeIndex]);
|
||||
nodes[nodeIndex] = node;
|
||||
return node;
|
||||
} finally {
|
||||
inProgress[nodeIndex] = false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void validateSerializedEdges(final int nodeIndex, final char... edgeLabels) throws IOException {
|
||||
for (int edgeIndex = 1; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||
if (edgeLabels[edgeIndex - 1] >= edgeLabels[edgeIndex]) {
|
||||
throw new IOException(
|
||||
"Edge labels must be strictly ascending at node " + nodeIndex + ", edge index " + edgeIndex
|
||||
+ ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -682,6 +915,16 @@ public final class FrequencyTrie<V> {
|
||||
* @return compiled node, or {@code null} if the path does not exist
|
||||
*/
|
||||
private CompiledNode<V> findNode(final String key) {
|
||||
return findNode((CharSequence) key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates the compiled node for the supplied key.
|
||||
*
|
||||
* @param key already-normalized key to resolve
|
||||
* @return compiled node, or {@code null} if the path does not exist
|
||||
*/
|
||||
private CompiledNode<V> findNode(final CharSequence key) {
|
||||
CompiledNode<V> current = this.root;
|
||||
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
|
||||
@@ -702,6 +945,77 @@ public final class FrequencyTrie<V> {
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates the compiled node for the supplied key slice.
|
||||
*
|
||||
* @param key already-normalized key storage
|
||||
* @param offset first character offset
|
||||
* @param length number of characters to read
|
||||
* @return compiled node, or {@code null} if the path does not exist
|
||||
*/
|
||||
private CompiledNode<V> findNode(final char[] key, final int offset, final int length) {
|
||||
CompiledNode<V> current = this.root;
|
||||
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
for (int traversalOffset = offset + length - 1; traversalOffset >= offset; traversalOffset--) {
|
||||
current = current.findChild(key[traversalOffset]);
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
final int endExclusive = offset + length;
|
||||
for (int traversalOffset = offset; traversalOffset < endExclusive; traversalOffset++) {
|
||||
current = current.findChild(key[traversalOffset]);
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Visits node-local values without allocating result containers.
|
||||
*
|
||||
* @param node resolved node, or {@code null}
|
||||
* @param sink value sink
|
||||
* @param maxResults maximum values to visit
|
||||
* @return number of visited values
|
||||
*/
|
||||
private int visitNode(final CompiledNode<V> node, final EntrySink<? super V> sink, final int maxResults) {
|
||||
if (node == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
final V[] orderedValues = node.orderedValues();
|
||||
final int valueCount = Math.min(orderedValues.length, maxResults);
|
||||
if (valueCount == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
final int[] orderedCounts = node.orderedCounts();
|
||||
int visited = 0;
|
||||
for (int rank = 0; rank < valueCount; rank++) {
|
||||
visited++;
|
||||
if (!sink.accept(orderedValues[rank], orderedCounts[rank], rank)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return visited;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates visitor maximum result count.
|
||||
*
|
||||
* @param maxResults maximum result count
|
||||
*/
|
||||
private static void validateMaxResults(final int maxResults) {
|
||||
if (maxResults < 0) {
|
||||
throw new IllegalArgumentException("maxResults must be non-negative.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies lookup-time case normalization according to persisted metadata.
|
||||
*
|
||||
@@ -709,11 +1023,21 @@ public final class FrequencyTrie<V> {
|
||||
* @return normalized key for trie traversal
|
||||
*/
|
||||
private String normalizeLookupKey(final String key) {
|
||||
return normalizeLookupKey((CharSequence) key).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies lookup-time normalization according to persisted metadata.
|
||||
*
|
||||
* @param key lookup key
|
||||
* @return normalized key for trie traversal
|
||||
*/
|
||||
private CharSequence normalizeLookupKey(final CharSequence key) {
|
||||
if (!this.lowercasesLookupKeys && !this.removeDiacritics) {
|
||||
return key;
|
||||
}
|
||||
|
||||
String normalized = key;
|
||||
String normalized = key.toString();
|
||||
if (this.lowercasesLookupKeys) {
|
||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
@@ -771,6 +1095,16 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private final DiacriticProcessingMode diacriticProcessingMode;
|
||||
|
||||
/**
|
||||
* Dense edge lookup span threshold.
|
||||
* <p>
|
||||
* This value controls a speed/memory trade-off during freezing: dense child
|
||||
* lookup tables are allocated only for nodes whose child labels fit in this
|
||||
* span.
|
||||
* </p>
|
||||
*/
|
||||
private final int maxExpandedIndex;
|
||||
|
||||
/**
|
||||
* Mutable root node.
|
||||
*/
|
||||
@@ -837,11 +1171,39 @@ public final class FrequencyTrie<V> {
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
||||
final DiacriticProcessingMode diacriticProcessingMode) {
|
||||
this(arrayFactory, reductionSettings, traversalDirection, caseProcessingMode, diacriticProcessingMode,
|
||||
CompiledNode.DEFAULT_MAX_EXPANDED_INDEX);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder with the provided settings, explicit traversal
|
||||
* direction, explicit case processing mode, explicit diacritic processing mode,
|
||||
* and an explicit dense child lookup threshold.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param caseProcessingMode dictionary case processing mode
|
||||
* @param diacriticProcessingMode dictionary diacritic processing mode
|
||||
* @param maxExpandedIndex dense lookup span override; zero disables
|
||||
* dense lookup. Larger values increase direct
|
||||
* indexing opportunities while potentially
|
||||
* increasing materialization memory in nodes
|
||||
* whose edge label span is within the limit.
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
||||
final DiacriticProcessingMode diacriticProcessingMode, final int maxExpandedIndex) {
|
||||
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
this.reductionSettings = Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
this.caseProcessingMode = Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
this.diacriticProcessingMode = Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||
if (maxExpandedIndex < 0) {
|
||||
throw new IllegalArgumentException("maxExpandedIndex must be non-negative.");
|
||||
}
|
||||
this.maxExpandedIndex = maxExpandedIndex;
|
||||
this.root = new MutableNode<>();
|
||||
}
|
||||
|
||||
@@ -942,7 +1304,7 @@ public final class FrequencyTrie<V> {
|
||||
* @throws IllegalArgumentException if {@code count} is less than {@code 1}
|
||||
*/
|
||||
public Builder<V> put(final String key, final V value, final int count) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
Objects.requireNonNull(key, ARG_KEY);
|
||||
Objects.requireNonNull(value, "value");
|
||||
|
||||
if (count < 1) { // NOPMD
|
||||
@@ -1098,7 +1460,7 @@ public final class FrequencyTrie<V> {
|
||||
}
|
||||
|
||||
final CompiledNode<V> frozen = new CompiledNode<>(edges, childNodes, localSummary.orderedValues(),
|
||||
localSummary.orderedCounts());
|
||||
this.maxExpandedIndex, localSummary.orderedCounts());
|
||||
cache.put(reducedNode, frozen);
|
||||
return frozen;
|
||||
}
|
||||
|
||||
@@ -119,11 +119,11 @@ public final class FrequencyTrieBuilders {
|
||||
* Copies one compiled node and all reachable descendants into the target
|
||||
* builder.
|
||||
*
|
||||
* @param node current compiled node
|
||||
* @param keyBuilder current key builder
|
||||
* @param node current compiled node
|
||||
* @param keyBuilder current key builder
|
||||
* @param builder target mutable builder
|
||||
* @param traversalDirection logical key traversal direction used by the source
|
||||
* @param <V> value type
|
||||
* @param <V> value type
|
||||
*/
|
||||
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
|
||||
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {
|
||||
|
||||
@@ -67,7 +67,7 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
* instance can still be used safely when needed.
|
||||
* </p>
|
||||
*/
|
||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||
@SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" })
|
||||
public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
@@ -121,6 +121,13 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
|
||||
|
||||
/**
|
||||
* Return value used by
|
||||
* {@link #applyTo(CharSequence, String, WordTraversalDirection, char[], int, int)}
|
||||
* when the caller-owned output range is too small for the transformed text.
|
||||
*/
|
||||
public static final int APPLY_INSUFFICIENT_CAPACITY = -1;
|
||||
|
||||
/**
|
||||
* Prefix used in unsupported NOOP patch argument exceptions.
|
||||
*/
|
||||
@@ -346,6 +353,78 @@ public final class PatchCommandEncoder {
|
||||
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command into a caller-owned output buffer.
|
||||
*
|
||||
* <p>
|
||||
* The output array is not retained. Capacity failure is reported by
|
||||
* {@link #APPLY_INSUFFICIENT_CAPACITY} and leaves the output range unchanged.
|
||||
* Malformed compatibility cases preserve the source exactly as
|
||||
* {@link #apply(String, String, WordTraversalDirection)} does.
|
||||
* </p>
|
||||
*
|
||||
* @param source original source text
|
||||
* @param patchCommand compact patch command
|
||||
* @param traversalDirection traversal direction used by the patch command
|
||||
* @param output caller-owned output storage
|
||||
* @param outputOffset first writable output offset
|
||||
* @param outputLength writable output capacity
|
||||
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
|
||||
* when {@code outputLength} is too small
|
||||
*/
|
||||
public static int applyTo(final CharSequence source, final String patchCommand,
|
||||
final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset,
|
||||
final int outputLength) {
|
||||
Objects.requireNonNull(source, "source");
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
Objects.requireNonNull(output, "output");
|
||||
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
|
||||
|
||||
final int sourceLength = source.length();
|
||||
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
|
||||
if (producedLength > outputLength) {
|
||||
return APPLY_INSUFFICIENT_CAPACITY;
|
||||
}
|
||||
applyToOutput(source, 0, sourceLength, patchCommand, traversalDirection, output, outputOffset, producedLength);
|
||||
return producedLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command from a caller-owned source slice into a
|
||||
* caller-owned output buffer.
|
||||
*
|
||||
* @param source source storage
|
||||
* @param sourceOffset first source character offset
|
||||
* @param sourceLength number of source characters
|
||||
* @param patchCommand compact patch command
|
||||
* @param traversalDirection traversal direction used by the patch command
|
||||
* @param output caller-owned output storage
|
||||
* @param outputOffset first writable output offset
|
||||
* @param outputLength writable output capacity
|
||||
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
|
||||
* when {@code outputLength} is too small
|
||||
* @throws IllegalArgumentException when source and output ranges overlap in the
|
||||
* same array
|
||||
*/
|
||||
public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength,
|
||||
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||
final int outputOffset, final int outputLength) {
|
||||
Objects.requireNonNull(source, "source");
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
Objects.requireNonNull(output, "output");
|
||||
Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length);
|
||||
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
|
||||
validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength);
|
||||
|
||||
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
|
||||
if (producedLength > outputLength) {
|
||||
return APPLY_INSUFFICIENT_CAPACITY;
|
||||
}
|
||||
applyToOutput(source, sourceOffset, sourceLength, patchCommand, traversalDirection, output, outputOffset,
|
||||
producedLength);
|
||||
return producedLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a patch command using the historical backward Egothor semantics.
|
||||
*
|
||||
@@ -409,7 +488,6 @@ public final class PatchCommandEncoder {
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
||||
private static String applyBackward(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
@@ -435,7 +513,7 @@ public final class PatchCommandEncoder {
|
||||
int position = result.length() - 1;
|
||||
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
@@ -493,7 +571,6 @@ public final class PatchCommandEncoder {
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
||||
private static String applyForward(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
@@ -519,7 +596,7 @@ public final class PatchCommandEncoder {
|
||||
int position = 0;
|
||||
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
@@ -681,7 +758,7 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
@@ -722,7 +799,7 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
@@ -753,6 +830,711 @@ public final class PatchCommandEncoder {
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the transformed length or the preserved source length for malformed
|
||||
* compatibility cases.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeAppliedLength(final int sourceLength, final String patchCommand,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|
||||
|| (patchCommand.length() & 1) != 0) {
|
||||
return sourceLength;
|
||||
}
|
||||
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
return computeBackwardAppliedLength(sourceLength, patchCommand);
|
||||
}
|
||||
return computeForwardAppliedLength(sourceLength, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the backward traversal output length.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeBackwardAppliedLength(final int sourceLength, final String patchCommand) {
|
||||
if (patchCommand.length() == 2) {
|
||||
return computeSingleBackwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||
}
|
||||
if (sourceLength == 0) {
|
||||
return computeBackwardEmptyAppliedLength(patchCommand);
|
||||
}
|
||||
|
||||
int currentLength = sourceLength;
|
||||
int position = sourceLength - 1;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return sourceLength;
|
||||
}
|
||||
position = position - skipCount + 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
if (position < 0 || position >= currentLength) {
|
||||
return sourceLength;
|
||||
}
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1) {
|
||||
return sourceLength;
|
||||
}
|
||||
final int deleteEndExclusive = position + 1;
|
||||
position -= deleteCount - 1;
|
||||
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
|
||||
return sourceLength;
|
||||
}
|
||||
currentLength -= deleteEndExclusive - position;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
if (position < -1 || position >= currentLength) {
|
||||
return sourceLength;
|
||||
}
|
||||
currentLength++;
|
||||
position++;
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return sourceLength;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
|
||||
position--;
|
||||
}
|
||||
return currentLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the forward traversal output length.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeForwardAppliedLength(final int sourceLength, final String patchCommand) {
|
||||
if (patchCommand.length() == 2) {
|
||||
return computeSingleForwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||
}
|
||||
if (sourceLength == 0) {
|
||||
return computeForwardEmptyAppliedLength(patchCommand);
|
||||
}
|
||||
|
||||
int currentLength = sourceLength;
|
||||
int position = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return sourceLength;
|
||||
}
|
||||
position = position + skipCount - 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
if (position < 0 || position >= currentLength) {
|
||||
return sourceLength;
|
||||
}
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
|
||||
return sourceLength;
|
||||
}
|
||||
currentLength -= deleteCount;
|
||||
position--;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
if (position < 0 || position > currentLength) {
|
||||
return sourceLength;
|
||||
}
|
||||
currentLength++;
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return sourceLength;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
|
||||
position++;
|
||||
}
|
||||
return currentLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a single backward instruction output length.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param opcode opcode
|
||||
* @param argument argument
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeSingleBackwardAppliedLength(final int sourceLength, final char opcode,
|
||||
final char argument) {
|
||||
final int encodedValue;
|
||||
switch (opcode) {
|
||||
case DELETE_OPCODE:
|
||||
encodedValue = decodeEncodedCount(argument);
|
||||
return encodedValue < 1 || encodedValue > sourceLength ? sourceLength : sourceLength - encodedValue;
|
||||
case INSERT_OPCODE:
|
||||
return sourceLength + 1;
|
||||
case REPLACE_OPCODE:
|
||||
case SKIP_OPCODE:
|
||||
return sourceLength;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return sourceLength;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a single forward instruction output length.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param opcode opcode
|
||||
* @param argument argument
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeSingleForwardAppliedLength(final int sourceLength, final char opcode,
|
||||
final char argument) {
|
||||
return computeSingleBackwardAppliedLength(sourceLength, opcode, argument);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes output length for an empty source in backward traversal.
|
||||
*
|
||||
* @param patchCommand patch command
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeBackwardEmptyAppliedLength(final String patchCommand) {
|
||||
int currentLength = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
switch (opcode) {
|
||||
case INSERT_OPCODE:
|
||||
currentLength++;
|
||||
break;
|
||||
case SKIP_OPCODE:
|
||||
case REPLACE_OPCODE:
|
||||
case DELETE_OPCODE:
|
||||
return 0;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return 0;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
return currentLength;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes output length for an empty source in forward traversal.
|
||||
*
|
||||
* @param patchCommand patch command
|
||||
* @return produced length
|
||||
*/
|
||||
private static int computeForwardEmptyAppliedLength(final String patchCommand) {
|
||||
return computeBackwardEmptyAppliedLength(patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies an already-sized patch into caller output.
|
||||
*
|
||||
* @param source source text
|
||||
* @param sourceOffset source offset
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
* @param producedLength already-validated produced length
|
||||
*/
|
||||
private static void applyToOutput(final CharSequence source, final int sourceOffset, final int sourceLength,
|
||||
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||
final int outputOffset, final int producedLength) {
|
||||
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
|
||||
copySource(source, sourceOffset, sourceLength, output, outputOffset);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sourceLength > 0) {
|
||||
copySource(source, sourceOffset, sourceLength, output, outputOffset);
|
||||
}
|
||||
|
||||
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||
} else {
|
||||
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies an already-sized patch into caller output.
|
||||
*
|
||||
* @param source source storage
|
||||
* @param sourceOffset source offset
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
* @param producedLength already-validated produced length
|
||||
*/
|
||||
private static void applyToOutput(final char[] source, final int sourceOffset, final int sourceLength,
|
||||
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||
final int outputOffset, final int producedLength) {
|
||||
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
|
||||
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sourceLength > 0) {
|
||||
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
|
||||
}
|
||||
|
||||
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||
} else {
|
||||
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether the output is exactly the original source.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param producedLength produced length
|
||||
* @param patchCommand patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @return {@code true} if copying the source is sufficient
|
||||
*/
|
||||
private static boolean isPreservedSource(final int sourceLength, final int producedLength,
|
||||
final String patchCommand, final WordTraversalDirection traversalDirection) {
|
||||
return producedLength == sourceLength
|
||||
&& isKnownPreserveOnlyPatch(sourceLength, patchCommand, traversalDirection);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether equal length also means no mutation is needed.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @return {@code true} when the command preserves source content
|
||||
*/
|
||||
private static boolean isKnownPreserveOnlyPatch(final int sourceLength, final String patchCommand,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|
||||
|| (patchCommand.length() & 1) != 0) {
|
||||
return true;
|
||||
}
|
||||
if (patchCommand.length() == 2) {
|
||||
return isSingleInstructionPreserveOnly(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||
}
|
||||
if (sourceLength == 0) {
|
||||
return hasEmptySourcePreserveOnlyPatch(patchCommand);
|
||||
}
|
||||
return traversalDirection == WordTraversalDirection.BACKWARD
|
||||
? hasBackwardPreserveOnlyPatch(sourceLength, patchCommand)
|
||||
: hasForwardPreserveOnlyPatch(sourceLength, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests whether a single instruction preserves the source content.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param opcode opcode
|
||||
* @param argument argument
|
||||
* @return {@code true} when no mutation should be applied
|
||||
*/
|
||||
private static boolean isSingleInstructionPreserveOnly(final int sourceLength, final char opcode,
|
||||
final char argument) {
|
||||
switch (opcode) {
|
||||
case DELETE_OPCODE:
|
||||
final int encodedValue = decodeEncodedCount(argument);
|
||||
return encodedValue < 1 || encodedValue > sourceLength;
|
||||
case INSERT_OPCODE:
|
||||
return false;
|
||||
case REPLACE_OPCODE:
|
||||
return sourceLength == 0;
|
||||
case SKIP_OPCODE:
|
||||
return true;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests whether an empty-source patch preserves the source.
|
||||
*
|
||||
* @param patchCommand patch command
|
||||
* @return {@code true} when no mutation should be applied
|
||||
*/
|
||||
private static boolean hasEmptySourcePreserveOnlyPatch(final String patchCommand) {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
switch (opcode) {
|
||||
case INSERT_OPCODE:
|
||||
break;
|
||||
case SKIP_OPCODE:
|
||||
case REPLACE_OPCODE:
|
||||
case DELETE_OPCODE:
|
||||
return true;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests whether a backward patch preserves the source because it is malformed
|
||||
* or a NOOP.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @return {@code true} when no mutation should be applied
|
||||
*/
|
||||
private static boolean hasBackwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
|
||||
int currentLength = sourceLength;
|
||||
int position = sourceLength - 1;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return true;
|
||||
}
|
||||
position = position - skipCount + 1;
|
||||
break;
|
||||
case REPLACE_OPCODE:
|
||||
if (position < 0 || position >= currentLength) {
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1) {
|
||||
return true;
|
||||
}
|
||||
final int deleteEndExclusive = position + 1;
|
||||
position -= deleteCount - 1;
|
||||
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
|
||||
return true;
|
||||
}
|
||||
currentLength -= deleteEndExclusive - position;
|
||||
break;
|
||||
case INSERT_OPCODE:
|
||||
if (position < -1 || position >= currentLength) {
|
||||
return true;
|
||||
}
|
||||
currentLength++;
|
||||
position++;
|
||||
break;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
position--;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests whether a forward patch preserves the source because it is malformed or
|
||||
* a NOOP.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @return {@code true} when no mutation should be applied
|
||||
*/
|
||||
private static boolean hasForwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
|
||||
int currentLength = sourceLength;
|
||||
int position = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return true;
|
||||
}
|
||||
position = position + skipCount - 1;
|
||||
break;
|
||||
case REPLACE_OPCODE:
|
||||
if (position < 0 || position >= currentLength) {
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
|
||||
return true;
|
||||
}
|
||||
currentLength -= deleteCount;
|
||||
position--;
|
||||
break;
|
||||
case INSERT_OPCODE:
|
||||
if (position < 0 || position > currentLength) {
|
||||
return true;
|
||||
}
|
||||
currentLength++;
|
||||
break;
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return true;
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
position++;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies source characters from a sequence.
|
||||
*
|
||||
* @param source source text
|
||||
* @param sourceOffset source offset
|
||||
* @param sourceLength source length
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
*/
|
||||
private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength,
|
||||
final char[] output, final int outputOffset) {
|
||||
for (int index = 0; index < sourceLength; index++) {
|
||||
output[outputOffset + index] = source.charAt(sourceOffset + index);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a backward patch after validation.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param output output storage initialized with source
|
||||
* @param outputOffset output offset
|
||||
*/
|
||||
private static void applyBackwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
|
||||
final int outputOffset) {
|
||||
if (sourceLength == 0) {
|
||||
applyBackwardEmptyToOutput(patchCommand, output, outputOffset);
|
||||
return;
|
||||
}
|
||||
|
||||
int currentLength = sourceLength;
|
||||
int position = sourceLength - 1;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
position = position - decodeEncodedCount(argument) + 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
output[outputOffset + position] = argument;
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteEndExclusive = position + 1;
|
||||
position -= decodeEncodedCount(argument) - 1;
|
||||
System.arraycopy(output, outputOffset + deleteEndExclusive, output, outputOffset + position,
|
||||
currentLength - deleteEndExclusive);
|
||||
currentLength -= deleteEndExclusive - position;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
final int insertIndex = position + 1;
|
||||
System.arraycopy(output, outputOffset + insertIndex, output, outputOffset + insertIndex + 1,
|
||||
currentLength - insertIndex);
|
||||
output[outputOffset + insertIndex] = argument;
|
||||
currentLength++;
|
||||
position++;
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
return;
|
||||
|
||||
default:
|
||||
throw new AssertionError("Patch command was not validated.");
|
||||
}
|
||||
|
||||
position--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a forward patch after validation.
|
||||
*
|
||||
* @param sourceLength source length
|
||||
* @param patchCommand patch command
|
||||
* @param output output storage initialized with source
|
||||
* @param outputOffset output offset
|
||||
*/
|
||||
private static void applyForwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
|
||||
final int outputOffset) {
|
||||
if (sourceLength == 0) {
|
||||
applyForwardEmptyToOutput(patchCommand, output, outputOffset);
|
||||
return;
|
||||
}
|
||||
|
||||
int currentLength = sourceLength;
|
||||
int position = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
position = position + decodeEncodedCount(argument) - 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
output[outputOffset + position] = argument;
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
System.arraycopy(output, outputOffset + position + deleteCount, output, outputOffset + position,
|
||||
currentLength - position - deleteCount);
|
||||
currentLength -= deleteCount;
|
||||
position--;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
System.arraycopy(output, outputOffset + position, output, outputOffset + position + 1,
|
||||
currentLength - position);
|
||||
output[outputOffset + position] = argument;
|
||||
currentLength++;
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
return;
|
||||
|
||||
default:
|
||||
throw new AssertionError("Patch command was not validated.");
|
||||
}
|
||||
|
||||
position++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies an empty-source backward patch after validation.
|
||||
*
|
||||
* @param patchCommand patch command
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
*/
|
||||
private static void applyBackwardEmptyToOutput(final String patchCommand, final char[] output,
|
||||
final int outputOffset) {
|
||||
int currentLength = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
System.arraycopy(output, outputOffset, output, outputOffset + 1, currentLength);
|
||||
output[outputOffset] = argument;
|
||||
currentLength++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies an empty-source forward patch after validation.
|
||||
*
|
||||
* @param patchCommand patch command
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
*/
|
||||
private static void applyForwardEmptyToOutput(final String patchCommand, final char[] output,
|
||||
final int outputOffset) {
|
||||
int currentLength = 0;
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||
output[outputOffset + currentLength] = patchCommand.charAt(patchIndex + 1);
|
||||
currentLength++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that source and output slices do not overlap when backed by the
|
||||
* same array.
|
||||
*
|
||||
* @param source source storage
|
||||
* @param sourceOffset source offset
|
||||
* @param sourceLength source length
|
||||
* @param output output storage
|
||||
* @param outputOffset output offset
|
||||
* @param outputLength output length
|
||||
*/
|
||||
private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset,
|
||||
final int sourceLength, final char[] output, final int outputOffset, final int outputLength) {
|
||||
if (!source.equals(output) || sourceLength == 0 || outputLength == 0) {
|
||||
return;
|
||||
}
|
||||
final int sourceEnd = sourceOffset + sourceLength;
|
||||
final int outputEnd = outputOffset + outputLength;
|
||||
if (sourceOffset < outputEnd && outputOffset < sourceEnd) {
|
||||
throw new IllegalArgumentException("source and output ranges must not overlap.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the direction-specialized apply strategy.
|
||||
*
|
||||
@@ -769,7 +1551,6 @@ public final class PatchCommandEncoder {
|
||||
* @param argument serialized count argument
|
||||
* @return decoded positive count, or {@code -1} when the argument is malformed
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
private static int decodeEncodedCount(final char argument) {
|
||||
if (argument < 'a') {
|
||||
return -1;
|
||||
|
||||
@@ -94,6 +94,29 @@ public final class StemmerPatchTrieBinaryIO {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a GZip-compressed binary patch-command trie from a filesystem path with
|
||||
* an optional dense child lookup span override.
|
||||
* <p>
|
||||
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||
* persisted in the file and does not change the compiled metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param path source file
|
||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @return deserialized trie
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
|
||||
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||
return read(fileInputStream, maxExpandedIndex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a GZip-compressed binary patch-command trie from a filesystem path
|
||||
* string.
|
||||
@@ -108,6 +131,26 @@ public final class StemmerPatchTrieBinaryIO {
|
||||
return read(Path.of(fileName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a GZip-compressed binary patch-command trie from a filesystem path
|
||||
* string with an optional dense child lookup span override.
|
||||
* <p>
|
||||
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||
* persisted in the file and does not change the compiled metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName source file name or path string
|
||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @return deserialized trie
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
return read(Path.of(fileName), maxExpandedIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a GZip-compressed binary patch-command trie from an input stream.
|
||||
*
|
||||
@@ -132,6 +175,35 @@ public final class StemmerPatchTrieBinaryIO {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a GZip-compressed binary patch-command trie from an input stream with
|
||||
* an optional dense child lookup span override.
|
||||
* <p>
|
||||
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||
* persisted in the file and does not change the compiled metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param inputStream source stream
|
||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @return deserialized trie
|
||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static FrequencyTrie<String> read(final InputStream inputStream, final int maxExpandedIndex)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
|
||||
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));
|
||||
DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) {
|
||||
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(dataInputStream, String[]::new, STRING_CODEC,
|
||||
maxExpandedIndex);
|
||||
|
||||
LOGGER.log(Level.FINE, "Read compressed binary stemmer trie.");
|
||||
return trie;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||
* at a filesystem path.
|
||||
|
||||
@@ -71,6 +71,7 @@ import java.util.zip.GZIPInputStream;
|
||||
public final class StemmerPatchTrieLoader {
|
||||
|
||||
/* default */ static final String FILENAME_REQUIRED = "fileName required";
|
||||
private static final String PARAMETER_PATH = "path";
|
||||
|
||||
/**
|
||||
* Logger of this class.
|
||||
@@ -461,7 +462,7 @@ public final class StemmerPatchTrieLoader {
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
|
||||
diacriticProcessingMode);
|
||||
return load(path, storeOriginal, metadata);
|
||||
@@ -487,7 +488,7 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
Objects.requireNonNull(metadata, "metadata");
|
||||
|
||||
try (InputStream inputStream = openDictionaryInputStream(path);
|
||||
@@ -759,10 +760,31 @@ public final class StemmerPatchTrieLoader {
|
||||
* read
|
||||
*/
|
||||
public static FrequencyTrie<String> loadBinary(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
return StemmerPatchTrieBinaryIO.read(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||
* using a custom dense lookup span override.
|
||||
* <p>
|
||||
* This is a runtime-only tuning parameter that does not affect persisted
|
||||
* metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the compressed binary trie file
|
||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static FrequencyTrie<String> loadBinary(final Path path, final int maxExpandedIndex) throws IOException {
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
return StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||
* string.
|
||||
@@ -778,6 +800,28 @@ public final class StemmerPatchTrieLoader {
|
||||
return StemmerPatchTrieBinaryIO.read(fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||
* string using a custom dense lookup span override.
|
||||
* <p>
|
||||
* This is a runtime-only tuning parameter that does not affect persisted
|
||||
* metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a GZip-compressed binary patch-command trie from an input stream.
|
||||
*
|
||||
@@ -802,7 +846,7 @@ public final class StemmerPatchTrieLoader {
|
||||
* read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(path);
|
||||
}
|
||||
|
||||
@@ -845,7 +889,7 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static void saveBinary(final FrequencyTrie<String> trie, final Path path) throws IOException {
|
||||
Objects.requireNonNull(trie, "trie");
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||
StemmerPatchTrieBinaryIO.write(trie, path);
|
||||
}
|
||||
|
||||
|
||||
@@ -58,17 +58,17 @@
|
||||
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
|
||||
* traditional line-oriented tab-separated values resource format in which each
|
||||
* non-empty logical line starts with a canonical stem followed by known surface
|
||||
* variants in subsequent tab-separated columns.
|
||||
* Parsing is delegated to {@link org.egothor.stemmer.StemmerDictionaryParser},
|
||||
* which applies configurable case processing through
|
||||
* variants in subsequent tab-separated columns. Parsing is delegated to
|
||||
* {@link org.egothor.stemmer.StemmerDictionaryParser}, which applies
|
||||
* configurable case processing through
|
||||
* {@link org.egothor.stemmer.CaseProcessingMode} (default:
|
||||
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
|
||||
* supports whole-line as well as trailing remarks introduced by {@code #} or
|
||||
* {@code //}, and currently ignores dictionary items containing Unicode
|
||||
* whitespace characters while reporting them through warning-level diagnostics.
|
||||
* During loading, each variant is converted into a patch command
|
||||
* targeting the canonical stem, and the stem itself may optionally be stored
|
||||
* under the canonical no-operation patch.
|
||||
* During loading, each variant is converted into a patch command targeting the
|
||||
* canonical stem, and the stem itself may optionally be stored under the
|
||||
* canonical no-operation patch.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
|
||||
@@ -1,21 +1,21 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
@@ -43,14 +43,15 @@ import java.util.Objects;
|
||||
* immutable from the public API perspective because construction wires these
|
||||
* arrays once and all lookup operations thereafter treat them as read-only.
|
||||
*
|
||||
* @param <V> value type
|
||||
* @param edgeLabels internal edge label array
|
||||
* @param children internal child array
|
||||
* @param orderedValues internal ordered values array
|
||||
* @param orderedCounts internal ordered counts array
|
||||
* @param <V> value type
|
||||
*/
|
||||
@SuppressWarnings("PMD.DataClass")
|
||||
public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[] orderedValues, int... orderedCounts) {
|
||||
public final class CompiledNode<V> {
|
||||
|
||||
/**
|
||||
* Default dense child lookup span in characters used when an explicit override
|
||||
* is not provided.
|
||||
*/
|
||||
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
|
||||
|
||||
/**
|
||||
* Number of child edges where linear scan is cheaper than binary search.
|
||||
@@ -58,24 +59,112 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4;
|
||||
|
||||
/**
|
||||
* Creates one validated compiled node.
|
||||
* Edge labels in sorted ascending order.
|
||||
*/
|
||||
private final char[] edgeLabels;
|
||||
|
||||
/**
|
||||
* Sparse child array aligned with {@link #edgeLabels}.
|
||||
*/
|
||||
private final CompiledNode<V>[] children;
|
||||
|
||||
/**
|
||||
* Dense child lookup table used when labels fit into a compact char interval.
|
||||
* <p>
|
||||
* The table enables direct O(1) indexing for child lookup and is allocated only
|
||||
* when the character span of this node's edges is within the configured
|
||||
* threshold.
|
||||
* </p>
|
||||
*/
|
||||
private final CompiledNode<V>[] denseChildren;
|
||||
|
||||
/**
|
||||
* Normalized minimum edge value for the dense lookup table.
|
||||
*/
|
||||
private final int denseEdgeMin;
|
||||
|
||||
/**
|
||||
* Values stored at this node in local order.
|
||||
*/
|
||||
private final V[] orderedValues;
|
||||
|
||||
/**
|
||||
* Occurrence counts aligned with {@link #orderedValues}.
|
||||
*/
|
||||
private final int[] orderedCounts;
|
||||
|
||||
/**
|
||||
* Creates one validated compiled node using {@link #DEFAULT_MAX_EXPANDED_INDEX}
|
||||
* for dense lookup sizing.
|
||||
*
|
||||
* @throws NullPointerException if any array argument is {@code null}
|
||||
* @throws IllegalArgumentException if the edge-related arrays or value-related
|
||||
* arrays do not have matching lengths
|
||||
*/
|
||||
public CompiledNode {
|
||||
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
|
||||
final int... orderedCounts) {
|
||||
this(edgeLabels, children, orderedValues, DEFAULT_MAX_EXPANDED_INDEX, orderedCounts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates one validated compiled node.
|
||||
*
|
||||
* @param maxExpandedIndex upper bound for the dense lookup interval size; zero
|
||||
* disables dense lookup. Larger values improve
|
||||
* direct-index likelihood while increasing dense table
|
||||
* memory in compact-label nodes.
|
||||
* @throws NullPointerException if any array argument is {@code null}
|
||||
* @throws IllegalArgumentException if the edge-related arrays or value-related
|
||||
* arrays do not have matching lengths or the
|
||||
* dense interval size is negative
|
||||
*/
|
||||
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
|
||||
final int maxExpandedIndex, final int... orderedCounts) {
|
||||
Objects.requireNonNull(edgeLabels, "edgeLabels");
|
||||
Objects.requireNonNull(children, "children");
|
||||
Objects.requireNonNull(orderedValues, "orderedValues");
|
||||
Objects.requireNonNull(orderedCounts, "orderedCounts");
|
||||
|
||||
if (maxExpandedIndex < 0) {
|
||||
throw new IllegalArgumentException("maxExpandedIndex must be non-negative.");
|
||||
}
|
||||
|
||||
if (edgeLabels.length != children.length) {
|
||||
throw new IllegalArgumentException("edgeLabels and children must have the same length.");
|
||||
}
|
||||
if (orderedValues.length != orderedCounts.length) {
|
||||
throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length.");
|
||||
}
|
||||
|
||||
this.edgeLabels = edgeLabels;
|
||||
this.children = children;
|
||||
this.orderedValues = orderedValues;
|
||||
this.orderedCounts = orderedCounts;
|
||||
|
||||
if (edgeLabels.length == 0 || maxExpandedIndex == 0) {
|
||||
this.denseChildren = null;
|
||||
this.denseEdgeMin = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
final int minEdge = edgeLabels[0];
|
||||
final int maxEdge = edgeLabels[edgeLabels.length - 1];
|
||||
final int span = maxEdge - minEdge;
|
||||
|
||||
if (span < 0 || span > maxExpandedIndex) {
|
||||
this.denseChildren = null;
|
||||
this.denseEdgeMin = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<V>[] dense = (CompiledNode<V>[]) new CompiledNode[span + 1];
|
||||
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
|
||||
}
|
||||
|
||||
this.denseChildren = dense;
|
||||
this.denseEdgeMin = minEdge;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -87,7 +176,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
*
|
||||
* @return internal edge-label array
|
||||
*/
|
||||
@Override
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public char[] edgeLabels() {
|
||||
return this.edgeLabels;
|
||||
@@ -102,7 +190,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
*
|
||||
* @return internal child-node array
|
||||
*/
|
||||
@Override
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public CompiledNode<V>[] children() {
|
||||
return this.children;
|
||||
@@ -117,7 +204,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
*
|
||||
* @return internal ordered-values array
|
||||
*/
|
||||
@Override
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public V[] orderedValues() {
|
||||
return this.orderedValues;
|
||||
@@ -132,14 +218,144 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
*
|
||||
* @return internal ordered-counts array
|
||||
*/
|
||||
@Override
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public int[] orderedCounts() {
|
||||
return this.orderedCounts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of child edges represented by this node.
|
||||
*
|
||||
* @return child edge count
|
||||
*/
|
||||
public int edgeCount() {
|
||||
return this.edgeLabels.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of values stored in this node.
|
||||
*
|
||||
* @return value count
|
||||
*/
|
||||
public int valueCount() {
|
||||
return this.orderedValues.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates whether this node stores any values.
|
||||
*
|
||||
* @return {@code true} when values are present at this node
|
||||
*/
|
||||
public boolean hasValues() {
|
||||
return this.orderedValues.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates whether this node has child edges.
|
||||
*
|
||||
* @return {@code true} when this node has at least one outgoing edge
|
||||
*/
|
||||
public boolean hasChildren() {
|
||||
return this.edgeLabels.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates whether this node has no child edges.
|
||||
*
|
||||
* @return {@code true} when this node is a terminal leaf node
|
||||
*/
|
||||
public boolean isLeaf() {
|
||||
return !hasChildren();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests whether an edge label is present at this node.
|
||||
*
|
||||
* @param edge edge label
|
||||
* @return {@code true} if this node contains the supplied edge label
|
||||
*/
|
||||
public boolean hasEdge(final char edge) {
|
||||
return findChild(edge) != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates whether this node has a dense direct-index child lookup table.
|
||||
*
|
||||
* @return {@code true} when a direct-index child table is available
|
||||
*/
|
||||
public boolean hasDenseLookup() {
|
||||
return this.denseChildren != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a small memory-related metric describing this node's dense table
|
||||
* size.
|
||||
*
|
||||
* @return number of dense table slots, or {@code 0} when dense lookup is not
|
||||
* enabled
|
||||
*/
|
||||
public int denseTableLength() {
|
||||
return this.denseChildren == null ? 0 : this.denseChildren.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a compact structural summary used by diagnostics and tests.
|
||||
*
|
||||
* @return summary hash for node structure and contents
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int hash = Arrays.hashCode(this.edgeLabels);
|
||||
hash = 31 * hash + Arrays.hashCode(this.children);
|
||||
hash = 31 * hash + Arrays.hashCode(this.orderedValues);
|
||||
hash = 31 * hash + Arrays.hashCode(this.orderedCounts);
|
||||
hash = 31 * hash + Objects.hash(this.denseEdgeMin);
|
||||
hash = 31 * hash + (hasDenseLookup() ? Arrays.hashCode(this.denseChildren) : 0);
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares structural node content, including dense table availability.
|
||||
*
|
||||
* @param object comparison object
|
||||
* @return {@code true} when nodes describe identical structure and payload
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object object) {
|
||||
if (this == object) {
|
||||
return true;
|
||||
}
|
||||
if (!(object instanceof CompiledNode<?> other)) {
|
||||
return false;
|
||||
}
|
||||
return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children)
|
||||
&& Arrays.equals(this.orderedValues, other.orderedValues)
|
||||
&& Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin
|
||||
&& Arrays.equals(this.denseChildren, other.denseChildren);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a short summary useful for debugging and diagnostics.
|
||||
*
|
||||
* @return textual node summary
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount="
|
||||
+ this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}';
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds a child for the supplied edge character.
|
||||
* <p>
|
||||
* Lookup order is:
|
||||
* <ol>
|
||||
* <li>dense array index (if the label interval is compact enough),</li>
|
||||
* <li>small-child linear scan when the fallback node has
|
||||
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
|
||||
* <li>binary search over sorted labels.</li>
|
||||
* </ol>
|
||||
* </p>
|
||||
*
|
||||
* @param edge edge character
|
||||
* @return child node, or {@code null} if absent
|
||||
@@ -149,6 +365,15 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
if (childCount == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this.denseChildren != null) {
|
||||
final int denseIndex = edge - this.denseEdgeMin;
|
||||
if (denseIndex < 0 || denseIndex >= this.denseChildren.length) {
|
||||
return null;
|
||||
}
|
||||
return this.denseChildren[denseIndex];
|
||||
}
|
||||
|
||||
if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) {
|
||||
for (int index = 0; index < childCount; index++) {
|
||||
if (this.edgeLabels[index] == edge) {
|
||||
|
||||
@@ -95,6 +95,9 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
@Tag("integration")
|
||||
@Tag("cli")
|
||||
@Tag("stemmer")
|
||||
@Tag("compile")
|
||||
@Tag("construction")
|
||||
@Tag("slow")
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
@DisplayName("Compile integration")
|
||||
final class CompileIntegrationTest {
|
||||
@@ -182,6 +185,11 @@ final class CompileIntegrationTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("Remark-aware fixture workflow")
|
||||
@Tag("integration")
|
||||
@Tag("cli")
|
||||
@Tag("stemmer")
|
||||
@Tag("compile")
|
||||
@Tag("construction")
|
||||
final class RemarkAwareFixtureWorkflow {
|
||||
|
||||
/**
|
||||
@@ -189,9 +197,10 @@ final class CompileIntegrationTest {
|
||||
* create nested output directories, preserve expected lookup behavior, and
|
||||
* store canonical stems when {@code --store-original} is enabled.
|
||||
*
|
||||
* @throws IOException if reading or writing fails
|
||||
* @throws IOException if reading or writing fails
|
||||
*/
|
||||
@Test
|
||||
@Tag("slow")
|
||||
@DisplayName("CLI should compile the remark-aware fixture and preserve expected lookups")
|
||||
void shouldCompileRemarkAwareFixtureAndPreserveExpectedLookups() throws IOException {
|
||||
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
||||
@@ -234,9 +243,10 @@ final class CompileIntegrationTest {
|
||||
* Verifies that the CLI rejects an already existing output path unless
|
||||
* overwrite is explicitly enabled.
|
||||
*
|
||||
* @throws IOException if reading or writing fails
|
||||
* @throws IOException if reading or writing fails
|
||||
*/
|
||||
@Test
|
||||
@Tag("slow")
|
||||
@DisplayName("CLI should require overwrite before replacing an existing output artifact")
|
||||
void shouldRequireOverwriteForExistingOutput() throws IOException {
|
||||
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
||||
@@ -301,6 +311,12 @@ final class CompileIntegrationTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("Bundled project dictionary workflows")
|
||||
@Tag("slow")
|
||||
@Tag("integration")
|
||||
@Tag("cli")
|
||||
@Tag("stemmer")
|
||||
@Tag("compile")
|
||||
@Tag("construction")
|
||||
final class BundledProjectDictionaryWorkflows {
|
||||
|
||||
/**
|
||||
@@ -317,11 +333,12 @@ final class CompileIntegrationTest {
|
||||
* </p>
|
||||
*
|
||||
* @param scenario scenario identifier
|
||||
* @param resourcePath bundled dictionary resource path
|
||||
* @param resourcePath bundled dictionary resource path
|
||||
* @throws IOException if reading or writing fails
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] {0}")
|
||||
@MethodSource("org.egothor.stemmer.CompileIntegrationTest#bundledDictionaryCases")
|
||||
@Tag("slow")
|
||||
@DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics")
|
||||
void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario,
|
||||
final String resourcePath) throws IOException {
|
||||
|
||||
@@ -66,7 +66,11 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
* {@link System#exit(int)}.
|
||||
* </p>
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("integration")
|
||||
@Tag("cli")
|
||||
@Tag("compile")
|
||||
@Tag("stemmer")
|
||||
@Tag("construction")
|
||||
@DisplayName("Compile")
|
||||
class CompileTest {
|
||||
|
||||
@@ -175,6 +179,11 @@ class CompileTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("argument validation")
|
||||
@Tag("integration")
|
||||
@Tag("cli")
|
||||
@Tag("compile")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
class ArgumentValidationTest {
|
||||
|
||||
@Test
|
||||
|
||||
@@ -70,10 +70,11 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
* <li>compressed artifact reproducibility within the active format version</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("compat")
|
||||
@Tag("regression")
|
||||
@Tag("determinism")
|
||||
@Tag("serialization")
|
||||
@Tag("trie")
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
final class CompiledTrieArtifactRegressionTest {
|
||||
|
||||
|
||||
@@ -41,7 +41,9 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link DiacriticStripper}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("diacritics")
|
||||
@Tag("diacritic")
|
||||
@Tag("stemmer")
|
||||
@Tag("normalization")
|
||||
@DisplayName("DiacriticStripper")
|
||||
class DiacriticStripperTest {
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ import org.junit.jupiter.api.Test;
|
||||
*/
|
||||
@DisplayName("FrequencyTrieBuilders")
|
||||
@Tag("unit")
|
||||
@Tag("builder")
|
||||
@Tag("construction")
|
||||
@Tag("frequency-trie")
|
||||
class FrequencyTrieBuildersTest {
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ import java.util.List;
|
||||
import net.jqwik.api.ForAll;
|
||||
import net.jqwik.api.Label;
|
||||
import net.jqwik.api.Property;
|
||||
import net.jqwik.api.Tag;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
|
||||
/**
|
||||
* Property-based tests for the compiled trie abstraction.
|
||||
@@ -59,9 +59,9 @@ import net.jqwik.api.Tag;
|
||||
* core algorithm without overfitting to particular fixture data.
|
||||
*/
|
||||
@Label("FrequencyTrie properties")
|
||||
@Tag("unit")
|
||||
@Tag("property")
|
||||
@Tag("trie")
|
||||
@Tag("frequency-trie")
|
||||
class FrequencyTrieProperties extends PropertyBasedTestSupport {
|
||||
|
||||
/**
|
||||
|
||||
@@ -33,6 +33,7 @@ package org.egothor.stemmer;
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
@@ -44,6 +45,7 @@ import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
@@ -62,6 +64,7 @@ import org.junit.jupiter.api.Test;
|
||||
@Tag("unit")
|
||||
@Tag("trie")
|
||||
@Tag("frequency-trie")
|
||||
@Tag("lookup")
|
||||
@DisplayName("FrequencyTrie")
|
||||
class FrequencyTrieTest {
|
||||
|
||||
@@ -379,6 +382,167 @@ class FrequencyTrieTest {
|
||||
assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that {@link FrequencyTrie#getEntries(String)} short-circuits to a one-item immutable list.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("getEntries returns a one-item list for single stored values")
|
||||
void getEntriesReturnsSingleItemListForSingleStoredValue() {
|
||||
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||
|
||||
builder.put("gamma", "only");
|
||||
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
final List<ValueCount<String>> entries = trie.getEntries("gamma");
|
||||
|
||||
assertAll(() -> assertEquals(List.of(new ValueCount<String>("only", 1)), entries),
|
||||
() -> assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that visitor lookup returns the same deterministic order and counts
|
||||
* as the allocating APIs.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Visitor lookup matches getAll order and getEntries counts")
|
||||
void visitorLookupMatchesGetAllOrderAndGetEntriesCounts() {
|
||||
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||
builder.put("house", "noun", 3);
|
||||
builder.put("house", "verb", 2);
|
||||
builder.put("house", "adjective", 1);
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
final List<String> values = new ArrayList<>();
|
||||
final List<Integer> counts = new ArrayList<>();
|
||||
final List<Integer> ranks = new ArrayList<>();
|
||||
|
||||
final int visited = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||
values.add(value);
|
||||
counts.add(count);
|
||||
ranks.add(rank);
|
||||
return true;
|
||||
}, 10);
|
||||
|
||||
assertAll(() -> assertEquals(3, visited),
|
||||
() -> assertEquals(List.of("noun", "verb", "adjective"), values),
|
||||
() -> assertEquals(List.of(3, 2, 1), counts),
|
||||
() -> assertEquals(List.of(0, 1, 2), ranks));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies visitor maximum result and early-stop behavior.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Visitor lookup honors maxResults and sink early stop")
|
||||
void visitorLookupHonorsMaxResultsAndSinkEarlyStop() {
|
||||
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||
builder.put("house", "noun", 3);
|
||||
builder.put("house", "verb", 2);
|
||||
builder.put("house", "adjective", 1);
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
final List<String> limited = new ArrayList<>();
|
||||
final List<String> stopped = new ArrayList<>();
|
||||
|
||||
final int limitedCount = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||
limited.add(value);
|
||||
return true;
|
||||
}, 2);
|
||||
final int stoppedCount = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||
stopped.add(value);
|
||||
return false;
|
||||
}, 10);
|
||||
|
||||
assertAll(() -> assertEquals(2, limitedCount),
|
||||
() -> assertEquals(List.of("noun", "verb"), limited),
|
||||
() -> assertEquals(1, stoppedCount),
|
||||
() -> assertEquals(List.of("noun"), stopped));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies visitor zero, negative, missing, and first-result behavior.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Visitor lookup handles zero, negative, missing, and first-result cases")
|
||||
void visitorLookupHandlesBoundaryCases() {
|
||||
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||
builder.put("house", "noun");
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
final int[] calls = new int[1];
|
||||
|
||||
assertAll(() -> assertEquals(0, trie.getAllNormalized("house", (value, count, rank) -> {
|
||||
calls[0]++;
|
||||
return true;
|
||||
}, 0)),
|
||||
() -> assertEquals(0, calls[0]),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> trie.getAllNormalized("house", (value, count, rank) -> true, -1)),
|
||||
() -> assertEquals(0, trie.getAllNormalized("missing", (value, count, rank) -> true, 10)),
|
||||
() -> assertFalse(trie.getFirstNormalized("missing", (value, count, rank) -> true)),
|
||||
() -> assertTrue(trie.getFirstNormalized("house", (value, count, rank) -> {
|
||||
assertEquals("noun", value);
|
||||
assertEquals(1, count);
|
||||
assertEquals(0, rank);
|
||||
return true;
|
||||
})));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies visitor API argument validation.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Visitor lookup rejects null and invalid range arguments")
|
||||
void visitorLookupRejectsNullAndInvalidRangeArguments() {
|
||||
final FrequencyTrie<String> trie = rankedBuilder().build();
|
||||
final char[] key = "house".toCharArray();
|
||||
final FrequencyTrie.EntrySink<String> sink = (value, count, rank) -> true;
|
||||
|
||||
assertAll(() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAllNormalized((char[]) null, 0, 0, sink, 1)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAllNormalized(key, 0, key.length, null, 1)),
|
||||
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||
() -> trie.getAllNormalized(key, -1, key.length, sink, 1)),
|
||||
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||
() -> trie.getAllNormalized(key, 1, key.length, sink, 1)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAllNormalized((CharSequence) null, sink, 1)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAllNormalized("house", null, 1)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAll((CharSequence) null, sink, 1)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> trie.getAll("house", null, 1)),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> trie.getAll("house", sink, -1)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies normalized char-array slices and metadata-aware CharSequence visitor
|
||||
* lookup.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Visitor lookup supports normalized char slices and metadata-aware CharSequence keys")
|
||||
void visitorLookupSupportsCharSlicesAndMetadataAwareKeys() {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
builder.put("house", "noun");
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
final char[] padded = "__house__".toCharArray();
|
||||
|
||||
assertAll(() -> assertEquals(1,
|
||||
trie.getAllNormalized(padded, 2, 5, (value, count, rank) -> {
|
||||
assertEquals("noun", value);
|
||||
return true;
|
||||
}, 10)),
|
||||
() -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true),
|
||||
"Normalized lookup must bypass metadata lowercasing."),
|
||||
() -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> {
|
||||
assertEquals("noun", value);
|
||||
return true;
|
||||
})));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that equal frequencies prefer the shorter string representation.
|
||||
*/
|
||||
@@ -755,6 +919,115 @@ class FrequencyTrieTest {
|
||||
.readFrom(new ByteArrayInputStream(serializedEmptyTrie), String[]::new, null)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that reading a compiled trie with a negative max-expanded override
|
||||
* smaller than -1 is rejected.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom rejects invalid maxExpandedIndex override")
|
||||
void readFromRejectsInvalidMaxExpandedIndexOverride() {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC, -2));
|
||||
|
||||
assertEquals("maxExpandedIndex must be >= -1.", exception.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the max-expanded override controls dense lookup materialization
|
||||
* while preserving lookup semantics.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom respects dense lookup max-expanded index override")
|
||||
void readFromRespectsDenseLookupMaxExpandedIndexOverride() throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||
|
||||
builder.put("a", "a");
|
||||
builder.put("b", "b");
|
||||
builder.put("c", "c");
|
||||
builder.put("d", "d");
|
||||
|
||||
final FrequencyTrie<String> original = builder.build();
|
||||
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
original.writeTo(outputStream, STRING_CODEC);
|
||||
final byte[] serializedTrie = outputStream.toByteArray();
|
||||
|
||||
final FrequencyTrie<String> defaultDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
|
||||
STRING_CODEC);
|
||||
final FrequencyTrie<String> defaultDenseByNegative = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie),
|
||||
String[]::new, STRING_CODEC, -1);
|
||||
final FrequencyTrie<String> disabledDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
|
||||
STRING_CODEC, 0);
|
||||
|
||||
assertAll(
|
||||
() -> assertTrue(defaultDense.root().hasDenseLookup(),
|
||||
"Default read should enable dense lookup for compact first-level edges."),
|
||||
() -> assertTrue(defaultDenseByNegative.root().hasDenseLookup(),
|
||||
"Negative override should use the default dense lookup span."),
|
||||
() -> assertFalse(disabledDense.root().hasDenseLookup(),
|
||||
"Zero override should disable dense lookup tables."),
|
||||
() -> assertEquals(original.get("a"), disabledDense.get("a")),
|
||||
() -> assertEquals(original.get("b"), disabledDense.get("b")),
|
||||
() -> assertEquals(original.get("c"), disabledDense.get("c")),
|
||||
() -> assertEquals(original.get("d"), disabledDense.get("d")),
|
||||
() -> assertEquals(original.get("z"), disabledDense.get("z")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that cyclic serialized node references are rejected as invalid
|
||||
* serialization.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom rejects cyclic serialized node references")
|
||||
void readFromRejectsCyclicSerializedNodeReferences() {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 1, 2, 0, new NodeWriter[] {
|
||||
dataOutput -> {
|
||||
dataOutput.writeInt(1);
|
||||
dataOutput.writeChar('b');
|
||||
dataOutput.writeInt(1);
|
||||
dataOutput.writeInt(0);
|
||||
},
|
||||
dataOutput -> {
|
||||
dataOutput.writeInt(1);
|
||||
dataOutput.writeChar('a');
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||
|
||||
assertTrue(exception.getMessage().contains("cyclic reference detected"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that child node references outside the valid serialized range are
|
||||
* rejected.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom rejects invalid child node identifiers")
|
||||
void readFromRejectsInvalidChildNodeId() {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(1);
|
||||
dataOutput.writeChar('a');
|
||||
dataOutput.writeInt(3);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||
|
||||
assertTrue(exception.getMessage().contains("Invalid child node id"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that deserialization rejects an invalid stream magic header.
|
||||
*/
|
||||
@@ -785,6 +1058,27 @@ class FrequencyTrieTest {
|
||||
assertTrue(exception.getMessage().contains("Unsupported trie stream version"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the latest stream version validates textual metadata blocks.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom rejects invalid textual metadata block")
|
||||
void readFromRejectsInvalidTextualMetadataBlock() {
|
||||
final int version = FrequencyTrie.currentFormatVersion();
|
||||
final byte[] bytes = createSerializedStream(0x45475452, version, 1, 0, dataOutput -> {
|
||||
dataOutput.writeUTF("not valid metadata");
|
||||
}, new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||
|
||||
assertTrue(exception.getMessage().contains("Invalid metadata block"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that deserialization rejects a negative node count.
|
||||
*/
|
||||
@@ -862,6 +1156,129 @@ class FrequencyTrieTest {
|
||||
assertTrue(exception.getMessage().contains("Non-positive stored count"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that legacy version 1 metadata uses compatibility defaults.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom supports legacy version 1 metadata")
|
||||
void readFromSupportsLegacyVersionOneMetadata() throws IOException {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||
|
||||
assertEquals(TrieMetadata.legacy(1, WordTraversalDirection.BACKWARD), trie.metadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that legacy version 2 metadata stores traversal direction and uses
|
||||
* compatibility defaults for other values.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom supports legacy version 2 metadata")
|
||||
void readFromSupportsLegacyVersionTwoMetadata() throws IOException {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
|
||||
dataOutput -> dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal()), new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||
|
||||
assertEquals(TrieMetadata.legacy(2, WordTraversalDirection.FORWARD), trie.metadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that version 3 metadata includes reduction and diacritic
|
||||
* processing settings.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom parses version 3 metadata")
|
||||
void readFromParsesVersionThreeMetadata() throws IOException {
|
||||
final ReductionSettings reductionSettings = new ReductionSettings(
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS, 81, 4);
|
||||
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 3, 1, 0,
|
||||
dataOutput -> {
|
||||
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
|
||||
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
|
||||
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
|
||||
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
|
||||
dataOutput.writeInt(DiacriticProcessingMode.REMOVE.ordinal());
|
||||
},
|
||||
new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||
final TrieMetadata metadata = trie.metadata();
|
||||
|
||||
assertAll(() -> assertEquals(3, metadata.formatVersion()),
|
||||
() -> assertEquals(WordTraversalDirection.BACKWARD, metadata.traversalDirection()),
|
||||
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
|
||||
() -> assertEquals(DiacriticProcessingMode.REMOVE, metadata.diacriticProcessingMode()),
|
||||
() -> assertEquals(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, metadata.caseProcessingMode()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that version 4 metadata additionally stores case-processing mode.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom parses version 4 case processing metadata")
|
||||
void readFromParsesVersionFourCaseMetadata() throws IOException {
|
||||
final ReductionSettings reductionSettings = new ReductionSettings(
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, 75, 3);
|
||||
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 4, 1, 0,
|
||||
dataOutput -> {
|
||||
dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal());
|
||||
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
|
||||
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
|
||||
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
|
||||
dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal());
|
||||
dataOutput.writeInt(CaseProcessingMode.AS_IS.ordinal());
|
||||
},
|
||||
new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||
final TrieMetadata metadata = trie.metadata();
|
||||
|
||||
assertAll(() -> assertEquals(4, metadata.formatVersion()),
|
||||
() -> assertEquals(WordTraversalDirection.FORWARD, metadata.traversalDirection()),
|
||||
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
|
||||
() -> assertEquals(DiacriticProcessingMode.AS_IS, metadata.diacriticProcessingMode()),
|
||||
() -> assertEquals(CaseProcessingMode.AS_IS, metadata.caseProcessingMode()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that invalid legacy metadata ordinals are rejected by validation.
|
||||
*/
|
||||
@Test
|
||||
@Tag("persistence")
|
||||
@DisplayName("readFrom rejects invalid metadata ordinal in legacy stream")
|
||||
void readFromRejectsInvalidLegacyMetadataOrdinal() {
|
||||
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
|
||||
dataOutput -> dataOutput.writeInt(999), new NodeWriter[] { dataOutput -> {
|
||||
dataOutput.writeInt(0);
|
||||
dataOutput.writeInt(0);
|
||||
} });
|
||||
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||
|
||||
assertTrue(exception.getMessage().contains("Invalid traversal direction ordinal"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one node body into a synthetic serialized trie stream.
|
||||
*/
|
||||
@@ -889,6 +1306,24 @@ class FrequencyTrieTest {
|
||||
*/
|
||||
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
|
||||
final int rootNodeId, final NodeWriter[] nodes) {
|
||||
return createSerializedStream(magic, version, nodeCount, rootNodeId, dataOutput -> {
|
||||
// legacy and text-based versions write their metadata differently.
|
||||
}, nodes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a synthetic serialized trie stream with a metadata writer hook.
|
||||
*
|
||||
* @param magic stream magic
|
||||
* @param version stream version
|
||||
* @param nodeCount declared node count
|
||||
* @param rootNodeId declared root node identifier
|
||||
* @param metadata version-specific metadata writer
|
||||
* @param nodes node body writers
|
||||
* @return serialized bytes
|
||||
*/
|
||||
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
|
||||
final int rootNodeId, final MetadataWriter metadata, final NodeWriter[] nodes) {
|
||||
try {
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
|
||||
@@ -897,6 +1332,7 @@ class FrequencyTrieTest {
|
||||
dataOutputStream.writeInt(version);
|
||||
dataOutputStream.writeInt(nodeCount);
|
||||
dataOutputStream.writeInt(rootNodeId);
|
||||
metadata.write(dataOutputStream);
|
||||
|
||||
for (NodeWriter node : nodes) {
|
||||
node.write(dataOutputStream);
|
||||
@@ -908,4 +1344,19 @@ class FrequencyTrieTest {
|
||||
throw new IllegalStateException("Unexpected I/O while building synthetic trie stream.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one synthetic metadata block.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
private interface MetadataWriter {
|
||||
|
||||
/**
|
||||
* Writes metadata bytes for one stream version.
|
||||
*
|
||||
* @param dataOutput output stream
|
||||
* @throws IOException if writing fails
|
||||
*/
|
||||
void write(DataOutputStream dataOutput) throws IOException;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,10 +65,10 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
* stems declared by the source dictionary.
|
||||
*/
|
||||
@DisplayName("Deterministic fuzz-style trie and stemmer compilation")
|
||||
@Tag("unit")
|
||||
@Tag("fuzz")
|
||||
@Tag("trie")
|
||||
@Tag("stemming")
|
||||
@Tag("stemmer")
|
||||
@Tag("determinism")
|
||||
class FuzzStemmerAndTrieCompilationTest {
|
||||
|
||||
/**
|
||||
|
||||
@@ -36,7 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import net.jqwik.api.ForAll;
|
||||
import net.jqwik.api.Label;
|
||||
import net.jqwik.api.Property;
|
||||
import net.jqwik.api.Tag;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
|
||||
/**
|
||||
* Property-based tests for {@link PatchCommandEncoder}.
|
||||
@@ -47,9 +47,9 @@ import net.jqwik.api.Tag;
|
||||
* reconstruct the exact requested target.
|
||||
*/
|
||||
@Label("PatchCommandEncoder properties")
|
||||
@Tag("unit")
|
||||
@Tag("property")
|
||||
@Tag("patch")
|
||||
@Tag("stemmer")
|
||||
class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||
|
||||
/**
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
@@ -68,6 +69,8 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
@Tag("encoding")
|
||||
@Tag("apply")
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
class PatchCommandEncoderTest {
|
||||
|
||||
@@ -147,6 +150,63 @@ class PatchCommandEncoderTest {
|
||||
Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides explicit forward-direction single-instruction patch application cases.
|
||||
*
|
||||
* @return test arguments
|
||||
*/
|
||||
private static Stream<Arguments> provideForwardSingleInstructionApplyCases() {
|
||||
return Stream.of(
|
||||
// 1
|
||||
Arguments.of(1, "abcd", "Db", "cd"),
|
||||
// 2
|
||||
Arguments.of(2, "abc", "Ia", "aabc"),
|
||||
// 3
|
||||
Arguments.of(3, "abc", "Ra", "abc"),
|
||||
// 4
|
||||
Arguments.of(4, "abc", "-a", "abc"),
|
||||
// 5
|
||||
Arguments.of(5, "abc", PatchCommandEncoder.NOOP_PATCH, "abc"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides forward-direction applyTo cases that exercise preserve-only and
|
||||
* non-preserve branches.
|
||||
*
|
||||
* @return test arguments
|
||||
*/
|
||||
private static Stream<Arguments> provideForwardApplyToCases() {
|
||||
return Stream.of(
|
||||
// 1
|
||||
Arguments.of(1, "book", "-aRa", "baok"),
|
||||
// 2
|
||||
Arguments.of(2, "abc", "-dRa", "abc"),
|
||||
// 3
|
||||
Arguments.of(3, "abc", "DdRa", "abc"),
|
||||
// 4
|
||||
Arguments.of(4, "abc", "-dIa", "abc"),
|
||||
// 5
|
||||
Arguments.of(5, "abc", "Na-a", "abc"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides empty-source forward applyTo cases that cover empty-source
|
||||
* instruction handling.
|
||||
*
|
||||
* @return test arguments
|
||||
*/
|
||||
private static Stream<Arguments> provideForwardEmptySourceApplyCases() {
|
||||
return Stream.of(
|
||||
// 1
|
||||
Arguments.of(1, "IaIb", "ab"),
|
||||
// 2
|
||||
Arguments.of(2, "-aRa", ""),
|
||||
// 3
|
||||
Arguments.of(3, "IaRa", ""),
|
||||
// 4
|
||||
Arguments.of(4, "Na-a", ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides malformed or index-invalid patch inputs that must preserve the
|
||||
* original source according to the implementation contract.
|
||||
@@ -236,12 +296,31 @@ class PatchCommandEncoderTest {
|
||||
return new StringBuilder(text).reverse().toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a patch into a right-sized output buffer and returns the produced
|
||||
* string.
|
||||
*
|
||||
* @param source source text
|
||||
* @param patch patch command
|
||||
* @param traversalDirection traversal direction
|
||||
* @return transformed text
|
||||
*/
|
||||
private static String applyToString(final String source, final String patch,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
final char[] output = new char[Math.max(source.length() + 16, 16)];
|
||||
final int produced = PatchCommandEncoder.applyTo(source, patch, traversalDirection, output, 0, output.length);
|
||||
return new String(output, 0, produced);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests constructor validation and basic instantiation behavior.
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("construction")
|
||||
@Tag("constructor")
|
||||
@Tag("construction")
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
class ConstructionTests {
|
||||
|
||||
/**
|
||||
@@ -326,7 +405,10 @@ class PatchCommandEncoderTest {
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("encode(String, String)")
|
||||
@Tag("encode")
|
||||
@Tag("encoding")
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
class EncodeTests {
|
||||
|
||||
/**
|
||||
@@ -461,6 +543,9 @@ class PatchCommandEncoderTest {
|
||||
@Nested
|
||||
@DisplayName("apply(String, String)")
|
||||
@Tag("apply")
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
class ApplyTests {
|
||||
|
||||
/**
|
||||
@@ -535,6 +620,101 @@ class PatchCommandEncoderTest {
|
||||
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit single-instruction forward patch application
|
||||
* semantics.
|
||||
*
|
||||
* @param caseId numeric case identifier
|
||||
* @param source source word
|
||||
* @param patch encoded patch command
|
||||
* @param expected expected transformed word
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: forward single instruction apply({1}, {2}) -> {3}")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardSingleInstructionApplyCases")
|
||||
@DisplayName("applies forward single instruction patches correctly")
|
||||
void shouldApplyForwardSingleInstructionsExplicitly(int caseId, String source, String patch, String expected) {
|
||||
assertEquals(expected, PatchCommandEncoder.apply(source, patch, WordTraversalDirection.FORWARD),
|
||||
() -> "Case " + caseId + " failed for source='" + source + "', patch='" + patch + "'.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies forward single-instruction malformed commands fail fast.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("throws for unsupported forward opcode and NOOP argument")
|
||||
void shouldThrowForUnsupportedForwardOpcodeAndNoopArgument() {
|
||||
assertAll(() -> assertEquals("Unsupported patch opcode: X",
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.apply("abc", "Xa", WordTraversalDirection.FORWARD))
|
||||
.getMessage()),
|
||||
() -> assertEquals("Unsupported NOOP patch argument: `",
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.apply("abc", "N`", WordTraversalDirection.FORWARD))
|
||||
.getMessage()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit forward-applyTo cases that exercise preserve-only and
|
||||
* non-preserve branches.
|
||||
*
|
||||
* @param caseId numeric case identifier
|
||||
* @param source source word
|
||||
* @param patch encoded patch command
|
||||
* @param expected expected transformed word
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: applyToForward({1}, {2}) -> {3}")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardApplyToCases")
|
||||
@DisplayName("applyTo handles forward preserve-only and mutation branches")
|
||||
void shouldApplyToForwardPreserveAndMutationBranches(int caseId, String source, String patch, String expected) {
|
||||
final char[] output = new char[Math.max(source.length() + 16, 16)];
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.FORWARD, output, 0,
|
||||
output.length);
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(expected.length(), produced,
|
||||
() -> "Case " + caseId + " produced wrong length."),
|
||||
() -> assertEquals(expected, new String(output, 0, produced),
|
||||
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies empty-source forward applyTo behavior for insert-only and malformed
|
||||
* instructions.
|
||||
*
|
||||
* @param caseId numeric case identifier
|
||||
* @param patch encoded patch command
|
||||
* @param expected expected transformed word
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: applyToForward(\"\", {1}) -> \"{2}\"")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardEmptySourceApplyCases")
|
||||
@DisplayName("applies forward empty-source patches correctly")
|
||||
void shouldApplyToForwardEmptySourceCases(int caseId, String patch, String expected) {
|
||||
final char[] output = new char[Math.max(expected.length() + 16, 16)];
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo("", patch, WordTraversalDirection.FORWARD, output, 0,
|
||||
output.length);
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(expected.length(), produced,
|
||||
() -> "Case " + caseId + " produced wrong length."),
|
||||
() -> assertEquals(expected, new String(output, 0, produced),
|
||||
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies malformed empty-source forward patches fail fast and preserve
|
||||
* empty-source semantics.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("throws for unsupported NOOP argument on empty-source forward patch")
|
||||
void shouldThrowForUnsupportedNoopArgumentOnForwardEmptySource() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.apply("", "N`Ra", WordTraversalDirection.FORWARD));
|
||||
|
||||
assertEquals("Unsupported NOOP patch argument: `", exception.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit patch application cases.
|
||||
*
|
||||
@@ -590,6 +770,181 @@ class PatchCommandEncoderTest {
|
||||
assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId
|
||||
+ " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies buffer application against string-returning application.
|
||||
*
|
||||
* @param caseId numeric case identifier
|
||||
* @param source source word
|
||||
* @param patch patch command
|
||||
* @param expected expected transformed word
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: applyTo({1}, {2}) -> {3}")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideApplyCases")
|
||||
@DisplayName("applyTo matches apply for explicit backward patch commands")
|
||||
void shouldApplyToBufferLikeApplyForBackwardCommands(int caseId, String source, String patch, String expected) {
|
||||
final char[] output = "___..............".toCharArray();
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.BACKWARD, output, 3,
|
||||
output.length - 3);
|
||||
|
||||
assertAll(() -> assertEquals(expected.length(), produced, () -> "Case " + caseId + " produced wrong length."),
|
||||
() -> assertEquals(expected, new String(output, 3, produced)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies char-array source slices.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo supports char-array source slices")
|
||||
void shouldApplyToCharArraySourceSlice() {
|
||||
final char[] source = "__teacher__".toCharArray();
|
||||
final char[] output = new char[16];
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo(source, 2, 7, "Db", WordTraversalDirection.BACKWARD,
|
||||
output, 1, output.length - 1);
|
||||
|
||||
assertAll(() -> assertEquals(5, produced), () -> assertEquals("teach", new String(output, 1, produced)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies null and range validation for buffer application.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo rejects null and invalid range arguments")
|
||||
void shouldRejectNullAndInvalidRangeArguments() {
|
||||
final char[] source = "teacher".toCharArray();
|
||||
final char[] output = new char[16];
|
||||
|
||||
assertAll(() -> assertThrows(NullPointerException.class,
|
||||
() -> PatchCommandEncoder.applyTo((CharSequence) null, "Db", WordTraversalDirection.BACKWARD,
|
||||
output, 0, output.length)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> PatchCommandEncoder.applyTo("teacher", "Db", null, output, 0, output.length)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD, null,
|
||||
0, output.length)),
|
||||
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD,
|
||||
output, -1, output.length)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> PatchCommandEncoder.applyTo((char[]) null, 0, 7, "Db",
|
||||
WordTraversalDirection.BACKWARD, output, 0, output.length)),
|
||||
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||
() -> PatchCommandEncoder.applyTo(source, 1, source.length, "Db",
|
||||
WordTraversalDirection.BACKWARD, output, 0, output.length)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies null, empty, and canonical NOOP patch preservation.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo preserves source for null, empty, and canonical NOOP patches")
|
||||
void shouldApplyToPreserveSourceForEmptyCompatibilityPatches() {
|
||||
final char[] nullPatchOutput = new char[8];
|
||||
final char[] emptyPatchOutput = new char[8];
|
||||
final char[] noopOutput = new char[8];
|
||||
|
||||
final int nullPatchLength = PatchCommandEncoder.applyTo("teacher", null, WordTraversalDirection.BACKWARD,
|
||||
nullPatchOutput, 0, nullPatchOutput.length);
|
||||
final int emptyPatchLength = PatchCommandEncoder.applyTo("teacher", "", WordTraversalDirection.BACKWARD,
|
||||
emptyPatchOutput, 0, emptyPatchOutput.length);
|
||||
final int noopLength = PatchCommandEncoder.applyTo("teacher", PatchCommandEncoder.NOOP_PATCH,
|
||||
WordTraversalDirection.BACKWARD, noopOutput, 0, noopOutput.length);
|
||||
|
||||
assertAll(() -> assertEquals(7, nullPatchLength),
|
||||
() -> assertEquals("teacher", new String(nullPatchOutput, 0, nullPatchLength)),
|
||||
() -> assertEquals(7, emptyPatchLength),
|
||||
() -> assertEquals("teacher", new String(emptyPatchOutput, 0, emptyPatchLength)),
|
||||
() -> assertEquals(7, noopLength),
|
||||
() -> assertEquals("teacher", new String(noopOutput, 0, noopLength)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies insufficient capacity behavior.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo reports insufficient capacity without writing output")
|
||||
void shouldReportInsufficientCapacityWithoutWritingOutput() {
|
||||
final char[] output = "xxxx".toCharArray();
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo("abc", "Ic", WordTraversalDirection.BACKWARD, output, 0,
|
||||
output.length - 1);
|
||||
|
||||
assertAll(() -> assertEquals(PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY, produced),
|
||||
() -> assertArrayEquals("xxxx".toCharArray(), output));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies exception parity with string-returning application.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo throws for unsupported opcode and NOOP argument")
|
||||
void shouldApplyToThrowForUnsupportedOpcodeAndNoopArgument() {
|
||||
final char[] output = new char[8];
|
||||
|
||||
assertAll(() -> {
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.applyTo("abc", "Xa", WordTraversalDirection.BACKWARD, output, 0,
|
||||
output.length));
|
||||
assertEquals("Unsupported patch opcode: X", exception.getMessage());
|
||||
}, () -> {
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.applyTo("abc", "Nb", WordTraversalDirection.BACKWARD, output, 0,
|
||||
output.length));
|
||||
assertEquals("Unsupported NOOP patch argument: b", exception.getMessage());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies malformed compatibility behavior for buffer application.
|
||||
*
|
||||
* @param caseId numeric case identifier
|
||||
* @param source original source
|
||||
* @param malformedPatch malformed patch
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: malformed applyTo patch {2} preserves {1}")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideMalformedPatchCases")
|
||||
@DisplayName("applyTo preserves source for malformed or index-invalid patch commands")
|
||||
void shouldApplyToPreserveSourceForMalformedOrIndexInvalidPatchCommands(int caseId, String source,
|
||||
String malformedPatch) {
|
||||
final char[] output = new char[Math.max(source.length(), 1)];
|
||||
|
||||
final int produced = PatchCommandEncoder.applyTo(source, malformedPatch, WordTraversalDirection.BACKWARD,
|
||||
output, 0, output.length);
|
||||
|
||||
assertAll(() -> assertEquals(source.length(), produced, () -> "Case " + caseId + " produced wrong length."),
|
||||
() -> assertEquals(source, new String(output, 0, produced)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit traversal direction for buffer application.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo follows explicit forward traversal direction")
|
||||
void shouldApplyToWithForwardTraversalDirection() {
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
final String patch = encoder.encode("cities", "city");
|
||||
|
||||
assertEquals(PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD),
|
||||
applyToString("cities", patch, WordTraversalDirection.FORWARD));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies overlapping source/output slices are rejected.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applyTo rejects overlapping char-array source and output ranges")
|
||||
void shouldRejectOverlappingSourceAndOutputRanges() {
|
||||
final char[] buffer = "teacher....".toCharArray();
|
||||
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> PatchCommandEncoder.applyTo(buffer, 0, 7, "Db", WordTraversalDirection.BACKWARD, buffer, 2, 5));
|
||||
|
||||
assertEquals("source and output ranges must not overlap.", exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -598,6 +953,9 @@ class PatchCommandEncoderTest {
|
||||
@Nested
|
||||
@DisplayName("stemming-oriented scenarios")
|
||||
@Tag("regression")
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
class StemmingScenarioTests {
|
||||
|
||||
/**
|
||||
@@ -658,7 +1016,10 @@ class PatchCommandEncoderTest {
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("reversed-word processing")
|
||||
@Tag("reverse")
|
||||
@Tag("normalization")
|
||||
@Tag("unit")
|
||||
@Tag("stemmer")
|
||||
@Tag("patch")
|
||||
class ReversedWordProcessingTests {
|
||||
|
||||
/**
|
||||
@@ -743,6 +1104,7 @@ class PatchCommandEncoderTest {
|
||||
*/
|
||||
@ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}")
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||
@Tag("normalization")
|
||||
@DisplayName("preserves correctness under mirrored input orientation")
|
||||
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
@@ -75,6 +75,8 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
@DisplayName("StemmerDictionaryParser")
|
||||
@Tag("unit")
|
||||
@Tag("parser")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
class StemmerDictionaryParserTest {
|
||||
|
||||
/**
|
||||
@@ -97,6 +99,10 @@ class StemmerDictionaryParserTest {
|
||||
/**
|
||||
* Log handler capturing parser diagnostics for assertions.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("parser")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
private static final class CapturedLogHandler extends Handler {
|
||||
|
||||
/**
|
||||
@@ -157,6 +163,10 @@ class StemmerDictionaryParserTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("parse(Reader, String, EntryHandler)")
|
||||
@Tag("unit")
|
||||
@Tag("parser")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
class ReaderParsingTests {
|
||||
|
||||
@Test
|
||||
@@ -333,6 +343,10 @@ class StemmerDictionaryParserTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("parse(Path, EntryHandler) and parse(String, EntryHandler)")
|
||||
@Tag("unit")
|
||||
@Tag("parser")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
class FileParsingTests {
|
||||
|
||||
@Test
|
||||
@@ -423,6 +437,10 @@ class StemmerDictionaryParserTest {
|
||||
|
||||
@Nested
|
||||
@DisplayName("ParseStatistics")
|
||||
@Tag("unit")
|
||||
@Tag("parser")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
class ParseStatisticsTests {
|
||||
|
||||
@Test
|
||||
|
||||
@@ -54,9 +54,10 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
/**
|
||||
* Tests for {@link StemmerKnowledgeExperiment}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("integration")
|
||||
@Tag("stemmer")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
final class StemmerKnowledgeExperimentTest {
|
||||
|
||||
/**
|
||||
|
||||
@@ -38,6 +38,8 @@ import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -91,6 +93,8 @@ import org.mockito.MockedStatic;
|
||||
@Tag("unit")
|
||||
@Tag("io")
|
||||
@Tag("persistence")
|
||||
@Tag("serialization")
|
||||
@Tag("trie")
|
||||
@DisplayName("StemmerPatchTrieBinaryIO")
|
||||
class StemmerPatchTrieBinaryIOTest {
|
||||
|
||||
@@ -130,6 +134,10 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("write(...)")
|
||||
@Tag("unit")
|
||||
@Tag("io")
|
||||
@Tag("trie")
|
||||
@Tag("persistence")
|
||||
class WriteTests {
|
||||
|
||||
/**
|
||||
@@ -286,6 +294,10 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("read(...)")
|
||||
@Tag("unit")
|
||||
@Tag("io")
|
||||
@Tag("trie")
|
||||
@Tag("persistence")
|
||||
class ReadTests {
|
||||
|
||||
/**
|
||||
@@ -299,9 +311,19 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
"read(Path) must reject null path."),
|
||||
() -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((String) null),
|
||||
"read(String) must reject null file name."),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> StemmerPatchTrieBinaryIO.read((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||
"read(Path, int) must reject null path."),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> StemmerPatchTrieBinaryIO.read((String) null,
|
||||
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||
"read(String, int) must reject null file name."),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null),
|
||||
"read(InputStream) must reject null input stream."));
|
||||
"read(InputStream) must reject null input stream."),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||
"read(InputStream, int) must reject null input stream."));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -385,6 +407,143 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that stream overload with dense span override delegates to the
|
||||
* four-argument readFrom method.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
@DisplayName("Should delegate stream read with dense span override")
|
||||
void shouldDelegateInputStreamReadWithDenseSpanOverride() throws IOException {
|
||||
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||
final byte[] gzipPayload = gzip("binary-content-with-max-expanded-index");
|
||||
|
||||
try (@SuppressWarnings("rawtypes")
|
||||
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||
|
||||
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO
|
||||
.read(new ByteArrayInputStream(gzipPayload), 17);
|
||||
|
||||
assertSame(expectedTrie, actualTrie,
|
||||
"read(InputStream, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||
|
||||
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), eq(17)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that path overload with dense span override delegates to the
|
||||
* same method overload with the override parameter.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
@DisplayName("Should delegate path read with dense span override")
|
||||
void shouldDelegatePathReadWithDenseSpanOverride() throws IOException {
|
||||
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||
final Path sourceFile = temporaryDirectory.resolve("input-max-expanded.bin.gz");
|
||||
Files.write(sourceFile, gzip("path-based-max-expanded-index"));
|
||||
|
||||
try (@SuppressWarnings("rawtypes")
|
||||
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||
|
||||
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile, 0);
|
||||
|
||||
assertSame(expectedTrie, actualTrie,
|
||||
"read(Path, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||
|
||||
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), eq(0)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that string path overload with dense span override delegates to the
|
||||
* same method overload with the override parameter.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
@DisplayName("Should delegate file name read with dense span override")
|
||||
void shouldDelegateStringReadWithDenseSpanOverride() throws IOException {
|
||||
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||
final Path sourceFile = temporaryDirectory.resolve("input-string-max-expanded.bin.gz");
|
||||
Files.write(sourceFile, gzip("string-based-max-expanded-index"));
|
||||
|
||||
try (@SuppressWarnings("rawtypes")
|
||||
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||
|
||||
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile.toString(), 32);
|
||||
|
||||
assertSame(expectedTrie, actualTrie,
|
||||
"read(String, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||
|
||||
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||
any(FrequencyTrie.ValueStreamCodec.class), eq(32)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that metadata-only read parses and returns the persisted metadata.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Should read metadata from gzip payload")
|
||||
void shouldReadMetadataFromGzipPayload() throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||
builder.put("run", PatchCommandEncoder.builder().build().encode("running", "run"));
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
StemmerPatchTrieBinaryIO.write(trie, outputStream);
|
||||
|
||||
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(new ByteArrayInputStream(outputStream.toByteArray()));
|
||||
|
||||
assertEquals(trie.metadata(), metadata,
|
||||
"readMetadata(InputStream) must return the same metadata persisted by write().");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that metadata can be read from a binary file path.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Should read metadata from file path")
|
||||
void shouldReadMetadataFromPath() throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
final Path sourceFile = temporaryDirectory.resolve("metadata-path.bin.gz");
|
||||
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
|
||||
|
||||
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile);
|
||||
assertEquals(trie.metadata(), metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that metadata can be read from a binary file name.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Should read metadata from file name")
|
||||
void shouldReadMetadataFromStringPath() throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
final Path sourceFile = temporaryDirectory.resolve("metadata-string.bin.gz");
|
||||
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
|
||||
|
||||
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile.toString());
|
||||
assertEquals(trie.metadata(), metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that malformed non-GZip input is reported as an I/O failure.
|
||||
*/
|
||||
@@ -482,6 +641,10 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
/**
|
||||
* Output stream that records whether it has been closed.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("io")
|
||||
@Tag("trie")
|
||||
@Tag("persistence")
|
||||
private static final class TrackingOutputStream extends ByteArrayOutputStream {
|
||||
|
||||
/**
|
||||
@@ -508,6 +671,10 @@ class StemmerPatchTrieBinaryIOTest {
|
||||
/**
|
||||
* Input stream that records whether it has been closed.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("io")
|
||||
@Tag("trie")
|
||||
@Tag("persistence")
|
||||
private static final class TrackingInputStream extends ByteArrayInputStream {
|
||||
|
||||
/**
|
||||
|
||||
@@ -85,9 +85,12 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
* <li>the current bundled language set, including right-to-left metadata</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("integration")
|
||||
@Tag("stemmer")
|
||||
@Tag("io")
|
||||
@Tag("parser")
|
||||
@Tag("trie")
|
||||
@Tag("persistence")
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
final class StemmerPatchTrieLoaderTest {
|
||||
|
||||
@@ -210,36 +213,43 @@ final class StemmerPatchTrieLoaderTest {
|
||||
Arguments.of("14-load-binary-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("15-load-binary-stream",
|
||||
Arguments.of("15-load-binary-path-override",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||
"path"),
|
||||
Arguments.of("16-load-binary-string-override",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null,
|
||||
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("17-load-binary-stream",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null),
|
||||
"inputStream"),
|
||||
Arguments.of("16-save-binary-null-trie-path",
|
||||
Arguments.of("18-save-binary-null-trie-path",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"),
|
||||
Arguments.of("17-save-binary-null-path",
|
||||
Arguments.of("19-save-binary-null-path",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"),
|
||||
Arguments.of("18-save-binary-null-trie-string",
|
||||
Arguments.of("20-save-binary-null-trie-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()),
|
||||
"trie"),
|
||||
Arguments.of("19-save-binary-null-string",
|
||||
Arguments.of("21-save-binary-null-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("20-load-language-null-metadata",
|
||||
Arguments.of("22-load-language-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||
true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("21-load-path-null-metadata",
|
||||
Arguments.of("23-load-path-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("22-load-string-null-metadata",
|
||||
Arguments.of("24-load-string-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||
(TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("23-load-binary-metadata-path-null",
|
||||
Arguments.of("25-load-binary-metadata-path-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
|
||||
Arguments.of("24-load-binary-metadata-string-null",
|
||||
Arguments.of("26-load-binary-metadata-string-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("25-load-binary-metadata-stream-null",
|
||||
Arguments.of("27-load-binary-metadata-stream-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
|
||||
"inputStream"));
|
||||
}
|
||||
@@ -258,6 +268,9 @@ final class StemmerPatchTrieLoaderTest {
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("API contracts")
|
||||
@Tag("validation")
|
||||
@Tag("integration")
|
||||
@Tag("trie")
|
||||
final class ApiContractTests {
|
||||
|
||||
/**
|
||||
@@ -306,11 +319,83 @@ final class StemmerPatchTrieLoaderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Focused internal loader behavior tests.
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("Internal helper behavior")
|
||||
@Tag("construction")
|
||||
@Tag("integration")
|
||||
@Tag("trie")
|
||||
final class InternalLoaderBehaviorTests {
|
||||
|
||||
/**
|
||||
* Verifies that bundled language loading follows explicit
|
||||
* right-to-left metadata mapping.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("bundled language loading must infer traversal direction from language metadata")
|
||||
void shouldLoadBundledLanguagesUsingLanguageRightToLeftMetadata() throws IOException {
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||
final FrequencyTrie<String> leftToRightDictionary = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK, true, settings);
|
||||
final FrequencyTrie<String> rightToLeftDictionary = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.FA_IR, true, settings);
|
||||
|
||||
assertEquals(WordTraversalDirection.BACKWARD, leftToRightDictionary.traversalDirection(),
|
||||
"Left-to-right languages should use backward traversal.");
|
||||
assertEquals(WordTraversalDirection.FORWARD, rightToLeftDictionary.traversalDirection(),
|
||||
"Right-to-left languages should use forward traversal.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the mode-based and settings-based bundled load overloads remain
|
||||
* semantically consistent for traversal direction.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("load(Language,.., ReductionMode) and load(Language,.., ReductionSettings) should agree on traversal direction")
|
||||
void shouldKeepBundledTraversalDirectionConsistentAcrossOverloads() throws IOException {
|
||||
final FrequencyTrie<String> byMode = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK, true, DEFAULT_REDUCTION_MODE);
|
||||
final FrequencyTrie<String> bySettings = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK, true,
|
||||
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE));
|
||||
|
||||
assertEquals(byMode.traversalDirection(), bySettings.traversalDirection());
|
||||
assertEquals(byMode.metadata().reductionSettings().reductionMode(),
|
||||
bySettings.metadata().reductionSettings().reductionMode());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies bundled resource access succeeds for known resources and fails
|
||||
* for unknown resources.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("openBundledResource should return readable streams for known resources and fail for unknown ones")
|
||||
void shouldOpenBundledResourcesSuccessfullyAndRejectUnknowns() throws IOException {
|
||||
final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath();
|
||||
try (InputStream inputStream = StemmerPatchTrieLoader.openBundledResource(resourcePath);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
assertNotNull(reader.readLine(), "Known bundled resource must expose readable content.");
|
||||
}
|
||||
|
||||
final String missingResource = "org/egothor/stemmer/missing-dictionary.dict.gz";
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> StemmerPatchTrieLoader.openBundledResource(missingResource));
|
||||
|
||||
assertEquals("Stemmer resource not found: " + missingResource, exception.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Focused filesystem and parser behavior tests.
|
||||
*/
|
||||
@Nested
|
||||
@DisplayName("Filesystem and parser behavior")
|
||||
@Tag("io")
|
||||
@Tag("construction")
|
||||
@Tag("integration")
|
||||
@Tag("trie")
|
||||
final class FilesystemAndParserTests {
|
||||
|
||||
/**
|
||||
@@ -512,6 +597,44 @@ final class StemmerPatchTrieLoaderTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that binary load overloads with an explicit dense lookup span
|
||||
* preserve trie semantics while honoring the dense-layout override.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Binary dense-span override overloads should load equivalent tries")
|
||||
void shouldLoadBinaryWithDenseSpanOverrideOverloads() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
run running runs runner
|
||||
city cities
|
||||
study studies studying
|
||||
""");
|
||||
final Path binaryFile = tempDir.resolve("stemmer-trie-overrides.bin.gz");
|
||||
|
||||
final FrequencyTrie<String> original = StemmerPatchTrieLoader.load(dictionaryFile, true,
|
||||
DEFAULT_REDUCTION_MODE);
|
||||
|
||||
StemmerPatchTrieLoader.saveBinary(original, binaryFile);
|
||||
|
||||
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
|
||||
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
|
||||
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX);
|
||||
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
|
||||
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
|
||||
|
||||
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
|
||||
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
|
||||
"studying");
|
||||
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
|
||||
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
|
||||
"studying");
|
||||
|
||||
assertFalse(fromPathNoDense.root().hasDenseLookup(),
|
||||
"Zero span should disable dense lookup on the loaded root.");
|
||||
assertFalse(fromStringNoDense.root().hasDenseLookup(),
|
||||
"Zero span should disable dense lookup on the loaded root.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a dictionary file into the temporary directory.
|
||||
*
|
||||
@@ -530,7 +653,11 @@ final class StemmerPatchTrieLoaderTest {
|
||||
* Bundled dictionary integration tests.
|
||||
*/
|
||||
@Nested
|
||||
@Tag("slow")
|
||||
@DisplayName("Bundled dictionaries")
|
||||
@Tag("compat")
|
||||
@Tag("trie")
|
||||
@Tag("regression")
|
||||
final class BundledDictionaryTests {
|
||||
|
||||
/**
|
||||
|
||||
@@ -44,7 +44,7 @@ import java.util.Set;
|
||||
import net.jqwik.api.ForAll;
|
||||
import net.jqwik.api.Label;
|
||||
import net.jqwik.api.Property;
|
||||
import net.jqwik.api.Tag;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
|
||||
/**
|
||||
* Property-based tests for patch-command stemmer tries.
|
||||
@@ -56,9 +56,8 @@ import net.jqwik.api.Tag;
|
||||
* persistence must not alter that behavior.
|
||||
*/
|
||||
@Label("Stemmer patch trie properties")
|
||||
@Tag("unit")
|
||||
@Tag("property")
|
||||
@Tag("stemming")
|
||||
@Tag("stemmer")
|
||||
class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
|
||||
|
||||
/**
|
||||
|
||||
@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@Tag("unit")
|
||||
@Tag("metadata")
|
||||
@Tag("trie")
|
||||
@Tag("validation")
|
||||
@DisplayName("TrieMetadata")
|
||||
class TrieMetadataTest {
|
||||
|
||||
|
||||
@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@Tag("unit")
|
||||
@Tag("core")
|
||||
@Tag("stemmer")
|
||||
@Tag("validation")
|
||||
@DisplayName("WordTraversalDirection")
|
||||
class WordTraversalDirectionTest {
|
||||
|
||||
|
||||
@@ -45,7 +45,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link ChildDescriptor}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("ChildDescriptor")
|
||||
class ChildDescriptorTest {
|
||||
|
||||
|
||||
@@ -31,8 +31,10 @@
|
||||
package org.egothor.stemmer.trie;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
@@ -43,11 +45,40 @@ import org.junit.jupiter.api.Test;
|
||||
* documented backing-array exposure.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("lookup")
|
||||
@DisplayName("CompiledNode and NodeData")
|
||||
class CompiledNodeAndNodeDataTest {
|
||||
|
||||
/**
|
||||
* Creates a typed child array for compiled-node tests.
|
||||
*
|
||||
* @param length requested array length
|
||||
* @return typed child array
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static CompiledNode<String>[] children(final int length) {
|
||||
return new CompiledNode[length];
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an empty child array for leaf compiled-node tests.
|
||||
*
|
||||
* @return empty typed child array
|
||||
*/
|
||||
private static CompiledNode<String>[] noChildren() {
|
||||
return children(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a leaf node used as a child in lookup tests.
|
||||
*
|
||||
* @return leaf node
|
||||
*/
|
||||
private static CompiledNode<String> leaf() {
|
||||
return new CompiledNode<>(new char[0], noChildren(), new String[0], new int[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that {@link NodeData} rejects mismatched edge-related array lengths.
|
||||
*/
|
||||
@@ -98,8 +129,7 @@ class CompiledNodeAndNodeDataTest {
|
||||
@Test
|
||||
@DisplayName("CompiledNode rejects mismatched edge and child arrays")
|
||||
void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() {
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<String>[] children = new CompiledNode[0];
|
||||
final CompiledNode<String>[] children = noChildren();
|
||||
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0]));
|
||||
@@ -113,8 +143,7 @@ class CompiledNodeAndNodeDataTest {
|
||||
@Test
|
||||
@DisplayName("CompiledNode rejects mismatched value arrays")
|
||||
void compiledNodeShouldRejectMismatchedValueArrays() {
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<String>[] children = new CompiledNode[0];
|
||||
final CompiledNode<String>[] children = noChildren();
|
||||
|
||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0]));
|
||||
@@ -130,8 +159,7 @@ class CompiledNodeAndNodeDataTest {
|
||||
@DisplayName("CompiledNode accessors expose documented backing arrays")
|
||||
void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() {
|
||||
final char[] edgeLabels = new char[] { 'a' };
|
||||
@SuppressWarnings("unchecked")
|
||||
final CompiledNode<String>[] children = new CompiledNode[1];
|
||||
final CompiledNode<String>[] children = children(1);
|
||||
final String[] orderedValues = new String[] { "stem" };
|
||||
final int[] orderedCounts = new int[] { 5 };
|
||||
final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts);
|
||||
@@ -141,4 +169,130 @@ class CompiledNodeAndNodeDataTest {
|
||||
assertSame(orderedValues, node.orderedValues());
|
||||
assertSame(orderedCounts, node.orderedCounts());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that dense lookup is used when the interval is compact.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("CompiledNode can resolve child via dense lookup table")
|
||||
void compiledNodeUsesDenseLookupForCompactIntervals() {
|
||||
final CompiledNode<String>[] children = children(4);
|
||||
children[0] = leaf();
|
||||
children[1] = leaf();
|
||||
children[2] = leaf();
|
||||
children[3] = leaf();
|
||||
|
||||
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'b', 'c', 'd' }, children,
|
||||
new String[] { "1", "2", "3", "4" }, new int[] { 1, 1, 1, 1 });
|
||||
|
||||
assertTrue(node.hasDenseLookup());
|
||||
|
||||
assertSame(children[0], node.findChild('a'));
|
||||
assertSame(children[3], node.findChild('d'));
|
||||
assertSame(null, node.findChild('z'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that fallback linear scan is used for small node degree.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("CompiledNode resolves child by linear scan for small degree")
|
||||
void compiledNodeUsesLinearScanForSmallDegree() {
|
||||
final CompiledNode<String>[] children = children(4);
|
||||
final CompiledNode<String> childA = leaf();
|
||||
final CompiledNode<String> childB = leaf();
|
||||
final CompiledNode<String> childC = leaf();
|
||||
final CompiledNode<String> childD = leaf();
|
||||
children[0] = childA;
|
||||
children[1] = childB;
|
||||
children[2] = childC;
|
||||
children[3] = childD;
|
||||
|
||||
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'z', '中', '你' }, children,
|
||||
new String[] { "1", "2", "3", "4" }, 0, new int[] { 1, 1, 1, 1 });
|
||||
|
||||
assertFalse(node.hasDenseLookup());
|
||||
|
||||
assertSame(childA, node.findChild('a'));
|
||||
assertSame(childD, node.findChild('你'));
|
||||
assertSame(null, node.findChild('b'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that fallback binary search is used for larger node degree without
|
||||
* dense lookup.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("CompiledNode resolves child by binary search for large degree")
|
||||
void compiledNodeUsesBinarySearchForLargeDegree() {
|
||||
final CompiledNode<String>[] children = children(5);
|
||||
final CompiledNode<String> childA = leaf();
|
||||
final CompiledNode<String> childB = leaf();
|
||||
final CompiledNode<String> childC = leaf();
|
||||
final CompiledNode<String> childD = leaf();
|
||||
final CompiledNode<String> childE = leaf();
|
||||
children[0] = childA;
|
||||
children[1] = childB;
|
||||
children[2] = childC;
|
||||
children[3] = childD;
|
||||
children[4] = childE;
|
||||
|
||||
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'c', 'k', 't', 'z' }, children,
|
||||
new String[] { "1", "2", "3", "4", "5" }, 0, new int[] { 1, 1, 1, 1, 1 });
|
||||
|
||||
assertFalse(node.hasDenseLookup());
|
||||
|
||||
assertSame(childC, node.findChild('k'));
|
||||
assertSame(childE, node.findChild('z'));
|
||||
assertSame(null, node.findChild('x'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the basic node-state helpers that are used by diagnostics and
|
||||
* behavioral checks.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("CompiledNode reports leaf, value and edge presence state")
|
||||
void compiledNodeReportsNodeStateHelpers() {
|
||||
final CompiledNode<String>[] childless = noChildren();
|
||||
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], childless, new String[0], new int[0]);
|
||||
|
||||
assertTrue(leaf.isLeaf());
|
||||
assertFalse(leaf.hasChildren());
|
||||
assertFalse(leaf.hasValues());
|
||||
assertFalse(leaf.hasEdge('a'));
|
||||
|
||||
final CompiledNode<String>[] child = children(1);
|
||||
final String[] orderedValues = new String[] { "leaf" };
|
||||
final int[] orderedCounts = new int[] { 1 };
|
||||
child[0] = new CompiledNode<>(new char[0], noChildren(), orderedValues, orderedCounts);
|
||||
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a' }, child, orderedValues, orderedCounts);
|
||||
|
||||
assertFalse(node.isLeaf());
|
||||
assertTrue(node.hasChildren());
|
||||
assertTrue(node.hasValues());
|
||||
assertTrue(node.valueCount() > 0);
|
||||
assertTrue(node.hasEdge('a'));
|
||||
assertFalse(node.hasEdge('b'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies structural equality and hash-code behavior for compiled nodes.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("CompiledNode equals and hashCode align for identical structure")
|
||||
void compiledNodeEqualsAndHashCodeAlignForIdenticalStructure() {
|
||||
final CompiledNode<String>[] child = children(1);
|
||||
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], noChildren(), new String[] { "v" },
|
||||
new int[] { 1 });
|
||||
child[0] = leaf;
|
||||
|
||||
final CompiledNode<String> first = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
|
||||
new int[] { 2 });
|
||||
final CompiledNode<String> second = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
|
||||
new int[] { 2 });
|
||||
|
||||
assertEquals(first, second);
|
||||
assertEquals(first.hashCode(), second.hashCode());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link DominantLocalDescriptor}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("DominantLocalDescriptor")
|
||||
class DominantLocalDescriptorTest {
|
||||
|
||||
|
||||
@@ -50,7 +50,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link LocalValueSummary}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("LocalValueSummary")
|
||||
class LocalValueSummaryTest {
|
||||
|
||||
@@ -190,6 +191,9 @@ class LocalValueSummaryTest {
|
||||
/**
|
||||
* Test helper with identical textual form but distinct identity.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
private static final class TextTwin {
|
||||
|
||||
/**
|
||||
|
||||
@@ -44,7 +44,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link MutableNode}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("construction")
|
||||
@DisplayName("MutableNode")
|
||||
class MutableNodeTest {
|
||||
|
||||
|
||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link RankedLocalDescriptor}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("RankedLocalDescriptor")
|
||||
class RankedLocalDescriptorTest {
|
||||
|
||||
|
||||
@@ -48,7 +48,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link ReducedNode}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("ReducedNode")
|
||||
class ReducedNodeTest {
|
||||
|
||||
|
||||
@@ -47,7 +47,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link ReductionContext}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("ReductionContext")
|
||||
class ReductionContextTest {
|
||||
|
||||
|
||||
@@ -46,7 +46,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link ReductionSignature}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("ReductionSignature")
|
||||
class ReductionSignatureTest {
|
||||
|
||||
|
||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
||||
* Unit tests for {@link UnorderedLocalDescriptor}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("fast")
|
||||
@Tag("trie")
|
||||
@Tag("reduction")
|
||||
@DisplayName("UnorderedLocalDescriptor")
|
||||
class UnorderedLocalDescriptorTest {
|
||||
|
||||
|
||||
Reference in New Issue
Block a user