Compare commits
4 Commits
release@2.
...
release@2.
| Author | SHA1 | Date | |
|---|---|---|---|
|
14a1e2fc53
|
|||
|
87ff85fd6d
|
|||
|
7bd0fc66ba
|
|||
|
dadab5514e
|
12
.github/workflows/build.yml
vendored
12
.github/workflows/build.yml
vendored
@@ -51,7 +51,7 @@ jobs:
|
|||||||
test -f gradle/verification-metadata.xml
|
test -f gradle/verification-metadata.xml
|
||||||
|
|
||||||
- name: Execute build, tests, PMD, coverage, Javadoc, distribution packaging, and SBOM generation
|
- name: Execute build, tests, PMD, coverage, Javadoc, distribution packaging, and SBOM generation
|
||||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport distZip cyclonedxBom
|
run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom
|
||||||
|
|
||||||
- name: Upload SBOM
|
- name: Upload SBOM
|
||||||
if: always()
|
if: always()
|
||||||
@@ -70,8 +70,8 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
name: test-reports
|
name: test-reports
|
||||||
path: |
|
path: |
|
||||||
build/reports/tests/test
|
build/reports/tests
|
||||||
build/test-results/test
|
build/test-results
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|
||||||
@@ -90,8 +90,8 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
name: coverage-reports
|
name: coverage-reports
|
||||||
path: |
|
path: |
|
||||||
build/reports/jacoco/test/html
|
build/reports/jacoco/jacocoCiReleaseReport/html
|
||||||
build/reports/jacoco/test/jacocoTestReport.xml
|
build/reports/jacoco/jacocoCiReleaseReport/jacocoCiReleaseReport.xml
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|
||||||
@@ -160,7 +160,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
|
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
|
||||||
SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
|
SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
|
||||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport cyclonedxBom centralBundle
|
run: ./gradlew --no-daemon clean ciRelease distZip pmdMain javadoc jacocoCiReleaseReport cyclonedxBom centralBundle
|
||||||
|
|
||||||
- name: Generate release changelog
|
- name: Generate release changelog
|
||||||
shell: bash
|
shell: bash
|
||||||
|
|||||||
19
.github/workflows/pages.yml
vendored
19
.github/workflows/pages.yml
vendored
@@ -70,7 +70,7 @@ jobs:
|
|||||||
test -f gradle/verification-metadata.xml
|
test -f gradle/verification-metadata.xml
|
||||||
|
|
||||||
- name: Build reports for publication
|
- name: Build reports for publication
|
||||||
run: ./gradlew --no-daemon clean build pmdMain javadoc jacocoTestReport pitest jmh cyclonedxBom
|
run: ./gradlew --no-daemon clean ciRelease pmdMain javadoc jacocoCiReleaseReport pitest jmh cyclonedxBom
|
||||||
|
|
||||||
- name: Prepare gh-pages worktree
|
- name: Prepare gh-pages worktree
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -93,6 +93,9 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
TEST_REPORT_DIR="build/reports/tests/ciRelease"
|
||||||
|
JACOCO_REPORT_DIR="build/reports/jacoco/jacocoCiReleaseReport"
|
||||||
|
|
||||||
SITE_DIR=".gh-pages"
|
SITE_DIR=".gh-pages"
|
||||||
RUN_DIR="${SITE_DIR}/builds/${GITHUB_RUN_NUMBER}"
|
RUN_DIR="${SITE_DIR}/builds/${GITHUB_RUN_NUMBER}"
|
||||||
RUN_METRICS_DIR="${RUN_DIR}/metrics"
|
RUN_METRICS_DIR="${RUN_DIR}/metrics"
|
||||||
@@ -106,14 +109,14 @@ jobs:
|
|||||||
cp -R build/docs/javadoc "${RUN_DIR}/javadoc"
|
cp -R build/docs/javadoc "${RUN_DIR}/javadoc"
|
||||||
cp -R build/docs/javadoc "${LATEST_DIR}/javadoc"
|
cp -R build/docs/javadoc "${LATEST_DIR}/javadoc"
|
||||||
|
|
||||||
cp -R build/reports/tests/test "${RUN_DIR}/test"
|
cp -R "${TEST_REPORT_DIR}" "${RUN_DIR}/test"
|
||||||
cp -R build/reports/tests/test "${LATEST_DIR}/test"
|
cp -R "${TEST_REPORT_DIR}" "${LATEST_DIR}/test"
|
||||||
|
|
||||||
cp -R build/reports/pmd "${RUN_DIR}/pmd"
|
cp -R build/reports/pmd "${RUN_DIR}/pmd"
|
||||||
cp -R build/reports/pmd "${LATEST_DIR}/pmd"
|
cp -R build/reports/pmd "${LATEST_DIR}/pmd"
|
||||||
|
|
||||||
cp -R build/reports/jacoco/test/html "${RUN_DIR}/coverage"
|
cp -R "${JACOCO_REPORT_DIR}/html" "${RUN_DIR}/coverage"
|
||||||
cp -R build/reports/jacoco/test/html "${LATEST_DIR}/coverage"
|
cp -R "${JACOCO_REPORT_DIR}/html" "${LATEST_DIR}/coverage"
|
||||||
|
|
||||||
cp -R build/reports/pitest "${RUN_DIR}/pitest"
|
cp -R build/reports/pitest "${RUN_DIR}/pitest"
|
||||||
cp -R build/reports/pitest "${LATEST_DIR}/pitest"
|
cp -R build/reports/pitest "${LATEST_DIR}/pitest"
|
||||||
@@ -178,7 +181,7 @@ jobs:
|
|||||||
|
|
||||||
python3 \
|
python3 \
|
||||||
./tools/generate-pages-badges.py \
|
./tools/generate-pages-badges.py \
|
||||||
--jacoco-xml build/reports/jacoco/test/jacocoTestReport.xml \
|
--jacoco-xml "${JACOCO_REPORT_DIR}/jacocoCiReleaseReport.xml" \
|
||||||
--pit-xml build/reports/pitest/mutations.xml \
|
--pit-xml build/reports/pitest/mutations.xml \
|
||||||
--jmh-csv build/reports/jmh/jmh-results.csv \
|
--jmh-csv build/reports/jmh/jmh-results.csv \
|
||||||
--run-metrics-dir "${RUN_METRICS_DIR}" \
|
--run-metrics-dir "${RUN_METRICS_DIR}" \
|
||||||
@@ -228,7 +231,7 @@ jobs:
|
|||||||
<p class="meta">Build ${GITHUB_RUN_NUMBER} from commit ${GITHUB_SHA}</p>
|
<p class="meta">Build ${GITHUB_RUN_NUMBER} from commit ${GITHUB_SHA}</p>
|
||||||
<ul>
|
<ul>
|
||||||
<li><a href="./javadoc/">Javadoc</a></li>
|
<li><a href="./javadoc/">Javadoc</a></li>
|
||||||
<li><a href="./test/">Test Report</a></li>
|
<li><a href="./test/">Release Verification Test Report (ciRelease)</a></li>
|
||||||
<li><a href="./pmd/main.html">PMD Report</a></li>
|
<li><a href="./pmd/main.html">PMD Report</a></li>
|
||||||
<li><a href="./coverage/">Coverage Report</a></li>
|
<li><a href="./coverage/">Coverage Report</a></li>
|
||||||
${DEPENDENCY_CHECK_LINK:-<li>Dependency Vulnerability Report: not available</li>}
|
${DEPENDENCY_CHECK_LINK:-<li>Dependency Vulnerability Report: not available</li>}
|
||||||
@@ -260,7 +263,7 @@ jobs:
|
|||||||
|
|
||||||
- [Latest build summary](https://leogalambos.github.io/Radixor/builds/latest/)
|
- [Latest build summary](https://leogalambos.github.io/Radixor/builds/latest/)
|
||||||
- [Javadoc](https://leogalambos.github.io/Radixor/builds/latest/javadoc/)
|
- [Javadoc](https://leogalambos.github.io/Radixor/builds/latest/javadoc/)
|
||||||
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
- [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||||
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
||||||
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||||
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||||
|
|||||||
27
.project
27
.project
@@ -2,21 +2,22 @@
|
|||||||
<projectDescription>
|
<projectDescription>
|
||||||
<name>Radixor</name>
|
<name>Radixor</name>
|
||||||
<comment></comment>
|
<comment></comment>
|
||||||
<projects/>
|
<projects>
|
||||||
|
</projects>
|
||||||
|
<buildSpec>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
</buildSpec>
|
||||||
<natures>
|
<natures>
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||||
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
||||||
</natures>
|
</natures>
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments/>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
|
||||||
<arguments/>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<linkedResources/>
|
|
||||||
<filteredResources/>
|
|
||||||
</projectDescription>
|
</projectDescription>
|
||||||
|
|||||||
@@ -167,6 +167,9 @@ The repository keeps the front page concise and places detailed documentation un
|
|||||||
- [Architecture](docs/architecture.md)
|
- [Architecture](docs/architecture.md)
|
||||||
Structural model, data flow, and runtime lookup behavior.
|
Structural model, data flow, and runtime lookup behavior.
|
||||||
|
|
||||||
|
- [Lookup Edge Optimization](docs/lookup-edge-optimization.md)
|
||||||
|
Speed/memory trade-off of dense child edge lookup in compiled tries.
|
||||||
|
|
||||||
- [Reduction Semantics](docs/reduction-semantics.md)
|
- [Reduction Semantics](docs/reduction-semantics.md)
|
||||||
Ranked, unordered, and dominant reduction behavior.
|
Ranked, unordered, and dominant reduction behavior.
|
||||||
|
|
||||||
|
|||||||
161
build.gradle
161
build.gradle
@@ -108,9 +108,19 @@ dependencyCheck {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.withType(Test).configureEach {
|
def cliIncludeTags = project.findProperty('includeTags')?.toString() ?: System.getProperty('includeTags')
|
||||||
useJUnitPlatform()
|
def cliExcludeTags = project.findProperty('excludeTags')?.toString() ?: System.getProperty('excludeTags')
|
||||||
|
|
||||||
|
def splitTagExpression = { String tagsExpr ->
|
||||||
|
if (tagsExpr == null || tagsExpr.isBlank()) {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
return tagsExpr.split(',')
|
||||||
|
.collect { it.trim() }
|
||||||
|
.findAll { it != null && !it.isBlank() }
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.withType(Test).configureEach {
|
||||||
doFirst {
|
doFirst {
|
||||||
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
||||||
}
|
}
|
||||||
@@ -123,14 +133,127 @@ tasks.withType(Test).configureEach {
|
|||||||
minHeapSize = '1g'
|
minHeapSize = '1g'
|
||||||
maxHeapSize = '4g'
|
maxHeapSize = '4g'
|
||||||
|
|
||||||
finalizedBy(tasks.named('jacocoTestReport'))
|
|
||||||
|
|
||||||
reports {
|
reports {
|
||||||
junitXml.required = true
|
junitXml.required = true
|
||||||
html.required = true
|
html.required = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def configureJUnitPlatformTags = { Test task, String includeTagsExpr, String excludeTagsExpr ->
|
||||||
|
task.useJUnitPlatform {
|
||||||
|
final def includes = splitTagExpression(includeTagsExpr)
|
||||||
|
final def excludes = splitTagExpression(excludeTagsExpr)
|
||||||
|
|
||||||
|
if (!includes.isEmpty()) {
|
||||||
|
includeTags(*includes.toArray(new String[0]))
|
||||||
|
}
|
||||||
|
if (!excludes.isEmpty()) {
|
||||||
|
excludeTags(*excludes.toArray(new String[0]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named('test', Test) {
|
||||||
|
configureJUnitPlatformTags(it, cliIncludeTags, cliExcludeTags)
|
||||||
|
finalizedBy(tasks.named('jacocoTestReport'))
|
||||||
|
}
|
||||||
|
|
||||||
|
def configureTaggedTestProfile = { String taskName, String includeTagsExpr, String excludeTagsExpr = null,
|
||||||
|
String taskDescription = null, String testNameExcludePatterns = null ->
|
||||||
|
tasks.register(taskName, Test) {
|
||||||
|
group = 'verification'
|
||||||
|
description = taskDescription
|
||||||
|
|
||||||
|
configureJUnitPlatformTags(delegate as Test, includeTagsExpr, excludeTagsExpr)
|
||||||
|
testClassesDirs = sourceSets.test.output.classesDirs
|
||||||
|
classpath = sourceSets.test.runtimeClasspath
|
||||||
|
dependsOn(tasks.named('compileTestJava'))
|
||||||
|
|
||||||
|
doFirst {
|
||||||
|
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (testNameExcludePatterns != null && !testNameExcludePatterns.isBlank()) {
|
||||||
|
filter {
|
||||||
|
testNameExcludePatterns.split(',').each { String pattern ->
|
||||||
|
final def trimmedPattern = pattern.trim()
|
||||||
|
if (!trimmedPattern.isEmpty()) {
|
||||||
|
excludeTestsMatching(trimmedPattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
minHeapSize = '1g'
|
||||||
|
maxHeapSize = '4g'
|
||||||
|
|
||||||
|
reports {
|
||||||
|
junitXml.required = true
|
||||||
|
html.required = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciSmoke',
|
||||||
|
'unit',
|
||||||
|
'slow',
|
||||||
|
'Fast feedback profile for unit tests with slow tests explicitly excluded.',
|
||||||
|
'org.egothor.stemmer.CompileIntegrationTest*'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciCore',
|
||||||
|
'unit,trie,frequency-trie,property',
|
||||||
|
null,
|
||||||
|
'Focused profile for core trie behavior and trie-specific property checks.'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciIntegration',
|
||||||
|
'integration',
|
||||||
|
'slow',
|
||||||
|
'Integration pipeline profile (loader/parser/CLI/IO end-to-end flows) excluding slow integration paths.'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciSlow',
|
||||||
|
'slow',
|
||||||
|
null,
|
||||||
|
'Targeted profile for all slow tests (large dictionaries, long-running corpus validation, and heavy integration checks).'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciCompat',
|
||||||
|
'compat,regression',
|
||||||
|
null,
|
||||||
|
'Compatibility profile guarding persisted artifact and compatibility regressions.'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciRelease',
|
||||||
|
null,
|
||||||
|
'slow',
|
||||||
|
'Release-profile validation of all non-slow tests.',
|
||||||
|
'org.egothor.stemmer.CompileIntegrationTest*,org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*'
|
||||||
|
)
|
||||||
|
|
||||||
|
configureTaggedTestProfile(
|
||||||
|
'ciNightly',
|
||||||
|
'fuzz',
|
||||||
|
null,
|
||||||
|
'Nightly robustness profile with fuzz testing emphasis.'
|
||||||
|
)
|
||||||
|
|
||||||
|
tasks.register('ci') {
|
||||||
|
group = 'verification'
|
||||||
|
description = 'Runs the full enterprise CI profile set in sequence.'
|
||||||
|
dependsOn(tasks.named('ciSmoke'))
|
||||||
|
dependsOn(tasks.named('ciCore'))
|
||||||
|
dependsOn(tasks.named('ciIntegration'))
|
||||||
|
dependsOn(tasks.named('ciCompat'))
|
||||||
|
}
|
||||||
|
|
||||||
tasks.withType(Pmd).configureEach {
|
tasks.withType(Pmd).configureEach {
|
||||||
reports {
|
reports {
|
||||||
xml.required = true
|
xml.required = true
|
||||||
@@ -155,6 +278,36 @@ tasks.named('jacocoTestReport', JacocoReport) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def registerJacocoProfileReport = { String reportTaskName, String sourceTaskName ->
|
||||||
|
tasks.register(reportTaskName, JacocoReport) {
|
||||||
|
group = 'verification'
|
||||||
|
description = "Generates Jacoco report for ${sourceTaskName} execution."
|
||||||
|
|
||||||
|
dependsOn(tasks.named(sourceTaskName))
|
||||||
|
|
||||||
|
classDirectories.setFrom(
|
||||||
|
files(sourceSets.main.output).asFileTree.matching {
|
||||||
|
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
|
||||||
|
exclude 'org/egothor/stemmer/DiacriticStripper*'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
executionData.setFrom(
|
||||||
|
fileTree(layout.buildDirectory.dir('jacoco')) {
|
||||||
|
include "${sourceTaskName}.exec"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
reports {
|
||||||
|
xml.required = true
|
||||||
|
csv.required = false
|
||||||
|
html.required = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
registerJacocoProfileReport('jacocoCiReleaseReport', 'ciRelease')
|
||||||
|
|
||||||
tasks.named('check') {
|
tasks.named('check') {
|
||||||
dependsOn(tasks.named('jacocoTestReport'))
|
dependsOn(tasks.named('jacocoTestReport'))
|
||||||
// no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze'))
|
// no-default, only on-demand: dependsOn(tasks.named('dependencyCheckAnalyze'))
|
||||||
|
|||||||
@@ -75,6 +75,14 @@ The distinction between preferred-result lookup and multi-result lookup is part
|
|||||||
|
|
||||||
That model is part of how the public API should be understood.
|
That model is part of how the public API should be understood.
|
||||||
|
|
||||||
|
Visitor lookup methods such as `getAllNormalized(..., EntrySink, maxResults)` are additive hot-path APIs. They expose the same local ordering and count semantics without allocating result containers, but they do not replace `get()`, `getAll()`, or `getEntries()`.
|
||||||
|
|
||||||
|
Compiled `FrequencyTrie` instances are immutable and thread-safe for concurrent reads. Visitor sinks are caller-owned and are not retained by the trie. Stored values passed to sinks are the model-owned trie values; for `FrequencyTrie<String>` patch tries, those patch strings are immutable stored strings rather than fresh per-result strings.
|
||||||
|
|
||||||
|
### Stable patch application behavior
|
||||||
|
|
||||||
|
`PatchCommandEncoder.apply(...)` remains the compatibility API for string-returning patch application. Buffer-oriented `applyTo(...)` overloads are additive APIs for caller-owned output storage. They do not retain output arrays, report insufficient capacity with `APPLY_INSUFFICIENT_CAPACITY`, and preserve the existing malformed-patch compatibility behavior where `apply(...)` preserves the source.
|
||||||
|
|
||||||
### Stable reduction-mode intent
|
### Stable reduction-mode intent
|
||||||
|
|
||||||
Each public `ReductionMode` constant carries a semantic contract that should remain meaningful across versions.
|
Each public `ReductionMode` constant carries a semantic contract that should remain meaningful across versions.
|
||||||
|
|||||||
193
docs/lookup-edge-optimization.md
Normal file
193
docs/lookup-edge-optimization.md
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
# Lookup Edge Optimization
|
||||||
|
|
||||||
|
Compiled trie nodes (`CompiledNode`) use three lookup strategies when resolving child edges:
|
||||||
|
|
||||||
|
1. dense array direct lookup,
|
||||||
|
2. linear scan for very small child counts,
|
||||||
|
3. binary search over sorted edge labels.
|
||||||
|
|
||||||
|
This page explains the dense path, what `maxExpandedIndex` controls, and how to tune it.
|
||||||
|
|
||||||
|
## Runtime model of one node
|
||||||
|
|
||||||
|
For a node with sorted edge labels `char[] edges`, the implementation can materialize an
|
||||||
|
index-aligned dense table when labels occupy a small compact code-point interval:
|
||||||
|
|
||||||
|
```text
|
||||||
|
span = maxEdge - minEdge
|
||||||
|
use dense table iff (span <= maxExpandedIndex) and (maxExpandedIndex > 0)
|
||||||
|
```
|
||||||
|
|
||||||
|
When dense lookup is used, lookup is constant-time indexing:
|
||||||
|
|
||||||
|
```text
|
||||||
|
denseIndex = requestedEdge - minEdge
|
||||||
|
return denseChildren[denseIndex] // or null if outside interval
|
||||||
|
```
|
||||||
|
|
||||||
|
When dense lookup is not active (interval is too wide or the configured
|
||||||
|
`maxExpandedIndex` is `0`), `CompiledNode` still chooses between two fallback
|
||||||
|
strategies:
|
||||||
|
|
||||||
|
- **linear scan** for very small child counts (`4` or fewer children),
|
||||||
|
- **binary search** for larger child counts.
|
||||||
|
|
||||||
|
This means the fallback method is selected by child count, not by “distance” alone.
|
||||||
|
`linear scan` is therefore used when there are only a few edges even if those edges are
|
||||||
|
spread across very distant code points.
|
||||||
|
|
||||||
|
### Example: few edges, wide Unicode span
|
||||||
|
|
||||||
|
```text
|
||||||
|
edges = ['a', '中', '你']
|
||||||
|
edge count = 3
|
||||||
|
minEdge = 'a' (U+0061)
|
||||||
|
maxEdge = '你' (U+4F60)
|
||||||
|
span = 20319
|
||||||
|
```
|
||||||
|
|
||||||
|
- If `maxExpandedIndex = 512`, dense indexing is not used because `span > maxExpandedIndex`.
|
||||||
|
- Because `edge count = 3` (<= 4), lookup falls back to a tiny linear scan of the
|
||||||
|
three labels.
|
||||||
|
- This is exactly the case where you get benefit from the threshold even though the interval is wide.
|
||||||
|
|
||||||
|
This is useful for non-Latin scripts as well: what matters is interval width in Unicode
|
||||||
|
code points, not script name. A compact Arabic-range block can still benefit from dense
|
||||||
|
lookups when keys stay in a tight code-point interval.
|
||||||
|
|
||||||
|
## Why this is configurable
|
||||||
|
|
||||||
|
`maxExpandedIndex` is only a performance/paging choice:
|
||||||
|
|
||||||
|
- higher value:
|
||||||
|
- more compact intervals qualify for dense tables,
|
||||||
|
- more constant-time child lookup,
|
||||||
|
- more memory for dense tables in qualifying nodes.
|
||||||
|
- lower value (or `0`):
|
||||||
|
- less dense-table allocation,
|
||||||
|
- fewer branches into constant-time path,
|
||||||
|
- lower materialization memory.
|
||||||
|
|
||||||
|
The value never changes lookup semantics. It only changes the in-memory structure shape.
|
||||||
|
|
||||||
|
## Persistence and loading model
|
||||||
|
|
||||||
|
This threshold is **not** stored in `TrieMetadata`.
|
||||||
|
|
||||||
|
- The binary format stores only trie payload and semantic metadata (`reduction`, `traversal`,
|
||||||
|
case/diacritic settings, and stream version).
|
||||||
|
- `maxExpandedIndex` is chosen when materializing nodes in memory.
|
||||||
|
- You can therefore keep one persisted artifact and load it with different in-memory
|
||||||
|
trade-offs depending on deployment constraints.
|
||||||
|
|
||||||
|
## Default
|
||||||
|
|
||||||
|
- `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX == 512`
|
||||||
|
- `CompiledNode.DEFAULT_MAX_EXPANDED_INDEX == 512`
|
||||||
|
|
||||||
|
These are practical defaults for mixed-language text and Latin-like scripts where edge labels
|
||||||
|
often cluster.
|
||||||
|
|
||||||
|
## Tune during build (writable phase)
|
||||||
|
|
||||||
|
Use the full `FrequencyTrie.Builder` constructor when you are compiling from source data.
|
||||||
|
The builder threshold is applied while freezing reduced nodes into the immutable form.
|
||||||
|
|
||||||
|
```java
|
||||||
|
import org.egothor.stemmer.CaseProcessingMode;
|
||||||
|
import org.egothor.stemmer.DiacriticProcessingMode;
|
||||||
|
import org.egothor.stemmer.FrequencyTrie;
|
||||||
|
import org.egothor.stemmer.ReductionMode;
|
||||||
|
import org.egothor.stemmer.ReductionSettings;
|
||||||
|
import org.egothor.stemmer.WordTraversalDirection;
|
||||||
|
|
||||||
|
final ReductionSettings settings = ReductionSettings.withDefaults(
|
||||||
|
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||||
|
|
||||||
|
final FrequencyTrie.Builder<String> fastBuilder =
|
||||||
|
new FrequencyTrie.Builder<>(String[]::new,
|
||||||
|
settings,
|
||||||
|
WordTraversalDirection.BACKWARD,
|
||||||
|
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||||
|
DiacriticProcessingMode.AS_IS,
|
||||||
|
1024); // prefer lookup speed
|
||||||
|
|
||||||
|
// ... put(...) ...
|
||||||
|
final FrequencyTrie<String> trie = fastBuilder.build();
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `0` or `256` for lower memory while still building larger tries.
|
||||||
|
|
||||||
|
```java
|
||||||
|
final FrequencyTrie.Builder<String> compactBuilder =
|
||||||
|
new FrequencyTrie.Builder<>(String[]::new,
|
||||||
|
settings,
|
||||||
|
WordTraversalDirection.BACKWARD,
|
||||||
|
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||||
|
DiacriticProcessingMode.AS_IS,
|
||||||
|
256); // lower memory profile
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tune when loading a binary artifact (runtime phase)
|
||||||
|
|
||||||
|
At artifact load time, you can tune the same trade-off independently of persisted metadata.
|
||||||
|
|
||||||
|
```java
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||||
|
|
||||||
|
var defaultLookup = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"));
|
||||||
|
|
||||||
|
var fastLookup = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"), 1024);
|
||||||
|
|
||||||
|
var compactLookup = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"), 0);
|
||||||
|
```
|
||||||
|
|
||||||
|
You can also set the threshold directly with `FrequencyTrie.readFrom(...)` when reading streams:
|
||||||
|
|
||||||
|
```java
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.zip.GZIPInputStream;
|
||||||
|
|
||||||
|
import org.egothor.stemmer.FrequencyTrie;
|
||||||
|
|
||||||
|
public final class StreamLoadExample {
|
||||||
|
|
||||||
|
private StreamLoadExample() {
|
||||||
|
throw new AssertionError("No instances.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(final String[] arguments) throws IOException {
|
||||||
|
try (InputStream fileInput = Files.newInputStream(Path.of("stemmers", "english.radixor.gz"));
|
||||||
|
GZIPInputStream gzip = new GZIPInputStream(fileInput);
|
||||||
|
DataInputStream dataInput = new DataInputStream(gzip)) {
|
||||||
|
final FrequencyTrie<String> compactOnLoad = FrequencyTrie.readFrom(
|
||||||
|
dataInput,
|
||||||
|
String[]::new,
|
||||||
|
input -> input.readUTF(),
|
||||||
|
256);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: the string codec is intentionally inline in this snippet to keep it self-contained.
|
||||||
|
|
||||||
|
## Practical guidance
|
||||||
|
|
||||||
|
- Start with default (`512`) in production and profile before changing it.
|
||||||
|
- Use `0` when memory is the priority and query throughput is not the bottleneck.
|
||||||
|
- Use values around `1024` for workloads dominated by compact alphabets and very hot lookups.
|
||||||
|
|
||||||
|
Trade-off expectation:
|
||||||
|
|
||||||
|
- increasing `maxExpandedIndex` improves lookup speed when edges tend to occupy short spans,
|
||||||
|
- decreasing it reduces per-node auxiliary memory in dense-span nodes.
|
||||||
@@ -87,6 +87,43 @@ public final class LoadBinaryExample {
|
|||||||
|
|
||||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
|
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
|
||||||
|
|
||||||
|
## Tune child lookup density when loading binaries
|
||||||
|
|
||||||
|
To optimize hot-path latency, you can tune direct child indexing by passing `maxExpandedIndex`
|
||||||
|
at load time. This does not change persisted metadata, only the materialized in-memory form.
|
||||||
|
|
||||||
|
```java
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import org.egothor.stemmer.FrequencyTrie;
|
||||||
|
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||||
|
|
||||||
|
public final class LoadBinaryWithDenseLookupExample {
|
||||||
|
|
||||||
|
private LoadBinaryWithDenseLookupExample() {
|
||||||
|
throw new AssertionError("No instances.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(final String[] arguments) throws IOException {
|
||||||
|
final FrequencyTrie<String> balanced = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"));
|
||||||
|
|
||||||
|
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"),
|
||||||
|
1024);
|
||||||
|
|
||||||
|
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"),
|
||||||
|
0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Negative values still use `FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX`.
|
||||||
|
|
||||||
|
[Lookup Edge Optimization](lookup-edge-optimization.md) describes the trade-off in detail and examples for build-time tuning as well.
|
||||||
|
|
||||||
## Build directly with a mutable builder
|
## Build directly with a mutable builder
|
||||||
|
|
||||||
A `FrequencyTrie.Builder<V>` accepts repeated `put(key, value)` calls and compiles the final read-only trie through `build()`. Compilation performs bottom-up reduction and produces the compact immutable runtime representation.
|
A `FrequencyTrie.Builder<V>` accepts repeated `put(key, value)` calls and compiles the final read-only trie through `build()`. Compilation performs bottom-up reduction and produces the compact immutable runtime representation.
|
||||||
|
|||||||
@@ -33,6 +33,28 @@ import org.egothor.stemmer.ValueCount;
|
|||||||
final List<ValueCount<String>> entries = trie.getEntries("axes");
|
final List<ValueCount<String>> entries = trie.getEntries("axes");
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Visitor lookup for hot paths
|
||||||
|
|
||||||
|
For allocation-sensitive token loops, use the visitor-style lookup methods. They visit the same ordered local values and counts without allocating a result array, list, or `ValueCount` objects.
|
||||||
|
|
||||||
|
```java
|
||||||
|
trie.getAll("axes", (patch, count, rank) -> {
|
||||||
|
// rank is zero-based and follows the same ordering as getAll(String).
|
||||||
|
return true; // return false to stop after this callback
|
||||||
|
}, 8);
|
||||||
|
```
|
||||||
|
|
||||||
|
If the caller has already normalized the input exactly as required by `trie.metadata()`, the normalized methods avoid lookup normalization buffers too:
|
||||||
|
|
||||||
|
```java
|
||||||
|
final char[] token = "axes".toCharArray();
|
||||||
|
trie.getAllNormalized(token, 0, token.length, (patch, count, rank) -> {
|
||||||
|
return true;
|
||||||
|
}, 8);
|
||||||
|
```
|
||||||
|
|
||||||
|
`getAllNormalized(...)` bypasses `caseProcessingMode` and `diacriticProcessingMode`; callers are responsible for supplying canonical input. `maxResults == 0` visits nothing, negative values are rejected, and a sink returning `false` stops iteration after the current callback.
|
||||||
|
|
||||||
## Apply patch commands
|
## Apply patch commands
|
||||||
|
|
||||||
A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch.
|
A patch command is not the final stem. It must be applied to the original input token. `PatchCommandEncoder.apply(source, patchCommand)` performs that transformation directly on the serialized command format. If the source is `null`, the method returns `null`. If the patch is `null`, empty, or malformed in compatibility-relevant ways, the original source word is preserved. Equal source and target words are represented by the canonical no-op patch.
|
||||||
@@ -45,6 +67,25 @@ final String patch = trie.get(word);
|
|||||||
final String stem = PatchCommandEncoder.apply(word, patch);
|
final String stem = PatchCommandEncoder.apply(word, patch);
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Hot paths can apply a patch into caller-owned character storage:
|
||||||
|
|
||||||
|
```java
|
||||||
|
final char[] output = new char[32];
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(
|
||||||
|
word,
|
||||||
|
patch,
|
||||||
|
trie.traversalDirection(),
|
||||||
|
output,
|
||||||
|
0,
|
||||||
|
output.length);
|
||||||
|
|
||||||
|
if (produced != PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
|
||||||
|
final String stem = new String(output, 0, produced);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`applyTo(...)` returns the produced character count on success and `APPLY_INSUFFICIENT_CAPACITY` when the output range is too small. Capacity failure does not write partial output. The source and output ranges of the `char[]` overload must not overlap.
|
||||||
|
|
||||||
For multiple candidates:
|
For multiple candidates:
|
||||||
|
|
||||||
```java
|
```java
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ This is why Radixor can generalize beyond explicitly listed forms and why compil
|
|||||||
The programmatic API is easier to understand when split by developer task:
|
The programmatic API is easier to understand when split by developer task:
|
||||||
|
|
||||||
- [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage.
|
- [Loading and Building Stemmers](programmatic-loading-and-building.md) explains how to acquire a compiled stemmer from bundled resources, textual dictionaries, binary artifacts, or direct builder usage.
|
||||||
|
- [Lookup Edge Optimization](lookup-edge-optimization.md) explains dense child lookup tuning and the speed/memory trade-off when materializing compiled tries.
|
||||||
- [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes.
|
- [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md) explains `get(...)`, `getAll(...)`, `getEntries(...)`, patch application, and the practical meaning of reduction modes.
|
||||||
- [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) explains how to reopen compiled tries, add new lexical data, rebuild them, and store them as binary artifacts.
|
- [Extending and Persisting Compiled Tries](programmatic-extending-and-persistence.md) explains how to reopen compiled tries, add new lexical data, rebuild them, and store them as binary artifacts.
|
||||||
|
|
||||||
|
|||||||
@@ -58,6 +58,27 @@ A deterministic system is easier to test, easier to reason about, and safer to i
|
|||||||
|
|
||||||
The project is intended to maintain very high confidence in both core correctness and behavioral stability.
|
The project is intended to maintain very high confidence in both core correctness and behavioral stability.
|
||||||
|
|
||||||
|
The recommended execution strategy is defined by the tagged test profiles in [Test taxonomy and execution filtering](test-taxonomy-and-filtering.md). In practice, teams can execute profile tasks directly:
|
||||||
|
|
||||||
|
- `./gradlew ciSmoke`: fast local/PR safety checks (`unit`, excluding `slow`; additionally excludes
|
||||||
|
`CompileIntegrationTest` as a defensive safeguard).
|
||||||
|
- `./gradlew ciSlow`: enterprise heavy gate for all tests marked with `slow` (typically
|
||||||
|
production dictionary and large corpus verification). This should be used for scheduled/manual
|
||||||
|
hardening gates and not in standard release build.
|
||||||
|
- `./gradlew ciCore`: behavioral coverage of trie and frequency-trie paths (`unit` + `property` where applicable)
|
||||||
|
- `./gradlew ciIntegration`: pipeline and CLI integration path checks
|
||||||
|
- `./gradlew ciCompat`: compatibility and regression verification for persisted artifacts
|
||||||
|
- `./gradlew ciRelease`: full non-slow suite for release-confidence runs (all test tags except `slow`,
|
||||||
|
plus explicit name-based exclusion of `CompileIntegrationTest*` and
|
||||||
|
`StemmerPatchTrieLoaderTest$BundledDictionaryTests*` as additional guardrails)
|
||||||
|
- `./gradlew ciNightly`: extended fuzz profile for robustness hardening
|
||||||
|
- `./gradlew ci`: umbrella profile depending on smoke/core/integration/compat
|
||||||
|
|
||||||
|
## Test taxonomy and execution filtering
|
||||||
|
|
||||||
|
The full tag taxonomy and executable filter examples are documented in
|
||||||
|
[Test taxonomy and execution filtering](test-taxonomy-and-filtering.md).
|
||||||
|
|
||||||
### Structural coverage
|
### Structural coverage
|
||||||
|
|
||||||
High code coverage is treated as a useful signal, but not as a sufficient goal on its own. Coverage is valuable only when the covered scenarios actually pressure the implementation in meaningful ways.
|
High code coverage is treated as a useful signal, but not as a sufficient goal on its own. Coverage is valuable only when the covered scenarios actually pressure the implementation in meaningful ways.
|
||||||
|
|||||||
@@ -67,6 +67,36 @@ public final class LoadBinaryStemmerExample {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
You can tune in-memory child lookup density at load time without changing the artifact:
|
||||||
|
|
||||||
|
```java
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import org.egothor.stemmer.FrequencyTrie;
|
||||||
|
import org.egothor.stemmer.StemmerPatchTrieLoader;
|
||||||
|
|
||||||
|
public final class LoadBinaryStemmerExampleTuned {
|
||||||
|
|
||||||
|
private LoadBinaryStemmerExampleTuned() {
|
||||||
|
throw new AssertionError("No instances.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(final String[] arguments) throws IOException {
|
||||||
|
final FrequencyTrie<String> fast = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"),
|
||||||
|
1024);
|
||||||
|
final FrequencyTrie<String> compact = StemmerPatchTrieLoader.loadBinary(
|
||||||
|
Path.of("stemmers", "english.radixor.gz"),
|
||||||
|
128);
|
||||||
|
|
||||||
|
System.out.println("fast=" + fast.size() + ", compact=" + compact.size());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
For the trade-off details, see [Lookup Edge Optimization](lookup-edge-optimization.md).
|
||||||
|
|
||||||
### Build or extend a stemmer from dictionary data
|
### Build or extend a stemmer from dictionary data
|
||||||
|
|
||||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.
|
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ These reports are primarily useful when reviewing the published API surface and
|
|||||||
|
|
||||||
These reports describe the outcome of core verification and static-analysis stages for the latest published build:
|
These reports describe the outcome of core verification and static-analysis stages for the latest published build:
|
||||||
|
|
||||||
- [Unit test report](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
- [Release verification test report (ciRelease)](https://leogalambos.github.io/Radixor/builds/latest/test/)
|
||||||
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
- [PMD report](https://leogalambos.github.io/Radixor/builds/latest/pmd/main.html)
|
||||||
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
- [JaCoCo coverage report](https://leogalambos.github.io/Radixor/builds/latest/coverage/)
|
||||||
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
- [PIT mutation testing report](https://leogalambos.github.io/Radixor/builds/latest/pitest/)
|
||||||
|
|||||||
216
docs/test-taxonomy-and-filtering.md
Normal file
216
docs/test-taxonomy-and-filtering.md
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
# Test Tag Taxonomy and Execution Guide
|
||||||
|
|
||||||
|
Radixor uses JUnit tags as an explicit execution policy for its test suite.
|
||||||
|
|
||||||
|
The project uses three orthogonal axes:
|
||||||
|
|
||||||
|
1. **Scope** (how the test is executed in the pipeline)
|
||||||
|
2. **Domain** (where in the system it belongs)
|
||||||
|
3. **Intent** (what behavior it verifies)
|
||||||
|
|
||||||
|
## Canonical scope tags
|
||||||
|
|
||||||
|
| Tag | Description | Typical usage |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `unit` | Fast, deterministic tests that exercise a specific class or behavior without external processes. | Default developer feedback; should stay near-zero flakiness and low run time. |
|
||||||
|
| `integration` | Tests that span multiple components or end-to-end flows of the public pipeline. | Parser/loader/CLI/IO integration checks and multi-step compile-then-load validations. |
|
||||||
|
| `property` | Property-based tests with generator-driven coverage for invariants. | Semantics-preserving laws and edge-case exploration beyond curated fixtures. |
|
||||||
|
| `fuzz` | Randomized stress checks with bounded runtime. | Heavier probabilistic verification of robustness and reduction invariants. |
|
||||||
|
| `compat` | Backward/forward compatibility and reproducibility checks for persisted artifacts. | Artifact fingerprints, deterministic rebuild, and regression fixtures. |
|
||||||
|
| `slow` | Long-running or expensive tests that should not execute in every fast gate. | Heavy fuzz/property budgets or high-duration integration checks. |
|
||||||
|
|
||||||
|
## Canonical domain tags
|
||||||
|
|
||||||
|
| Tag | Description | Typical usage |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `core` | Core algorithm and foundational platform behavior. | Traversal direction, base data structures, low-level helpers. |
|
||||||
|
| `trie` | All mutable/compiled trie behaviors and traversal internals. | Lookup path selection, node shape, child representation, subtree behavior. |
|
||||||
|
| `frequency-trie` | Algorithms and corner cases specific to frequency-aware trie logic. | Ranking, weighted reductions, persistence of weighted nodes. |
|
||||||
|
| `stemmer` | End-user stemming pipeline semantics. | Parse-encode-apply flows and output invariants. |
|
||||||
|
| `patch` | Patch encoding, decoding, and application semantics. | `PatchCommandEncoder` behavior and related compatibility contracts. |
|
||||||
|
| `io` | Input/output and resource loading boundaries. | Filesystem readers, streams, and stream lifecycle handling. |
|
||||||
|
| `serialization` | Binary persistence contract of compiled artifacts. | Versioned format reads/writes and checksum/consistency checks. |
|
||||||
|
| `parser` | Dictionary and metadata parsing concerns. | Dictionary input parsing and malformed-source rejection. |
|
||||||
|
| `cli` | Command-line entrypoint and command orchestration behavior. | Compile CLI integration and CLI argument validation. |
|
||||||
|
| `metadata` | Trie metadata semantics, compatibility fields, and schema expectations. | Version flags, structural properties, and metadata round-trips. |
|
||||||
|
| `compile` | Compile-time pipeline and build-oriented behavior. | Building, reduction-mode behavior, and compiled artifact generation. |
|
||||||
|
| `diacritic` | Unicode diacritic normalization and stripping behavior. | Accent-removal correctness and locale-safe normalization checks. |
|
||||||
|
|
||||||
|
## Canonical intent tags
|
||||||
|
|
||||||
|
| Tag | Description | Typical usage |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `construction` | Tests around construction and assembly of runtime structures. | Builders, loaders, and compile-time object construction contracts. |
|
||||||
|
| `lookup` | Read behavior and retrieval semantics. | `get()`, `getAll()`, traversal and missing-key behavior. |
|
||||||
|
| `persistence` | Storage lifecycle semantics. | Serialization/deserialization and round-trip correctness. |
|
||||||
|
| `reduction` | Reduction algorithm correctness and corner cases. | Dominance threshold, subtree deduplication, rank-preservation invariants. |
|
||||||
|
| `encoding` | Encoding transformation direction. | `PatchCommandEncoder.encode` and serialized command form generation. |
|
||||||
|
| `decoding` | Decoding/interpretation of persisted or runtime commands. | Optional consumers that parse and apply encoded command payloads. |
|
||||||
|
| `apply` | Patch application and transformation behavior. | Verifies that applied patches produce expected derived forms. |
|
||||||
|
| `normalization` | Canonicalization and cleanup behavior. | String normalization around case/shape and mirrored input paths. |
|
||||||
|
| `validation` | Input rejection and defensive checks. | Null/empty/invalid contracts and explicit failure conditions. |
|
||||||
|
| `regression` | Guard tests for behavior changes over time. | Known historical bugs and behavioral drift prevention. |
|
||||||
|
| `determinism` | Repeatable results under fixed input and settings. | Compile determinism, stable ordering, and artifact reproducibility. |
|
||||||
|
| `error-handling` | Exception surface and robustness expectations. | Recovery/failure modes and diagnostics quality. |
|
||||||
|
|
||||||
|
## Class-level rules
|
||||||
|
|
||||||
|
1. Every test class has **exactly one** scope tag.
|
||||||
|
2. Every test class has at least one domain tag.
|
||||||
|
3. Additional tags describe intent and may be used on classes or nested tests.
|
||||||
|
4. For each test class, intent tags should reflect the primary behavior under test, not historical naming conventions.
|
||||||
|
|
||||||
|
## Governance and execution policy
|
||||||
|
|
||||||
|
The following rules are used to keep the suite auditable and stable:
|
||||||
|
|
||||||
|
| Rule | Required state | Why |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Scope discipline | Exactly one scope tag per class. | Prevents accidental promotion of integration-only behavior into fast unit runs. |
|
||||||
|
| Coverage breadth | At least one domain tag per class. | Ensures tests can be grouped by subsystem for targeted review. |
|
||||||
|
| Intent specificity | Use at least one intent tag when behavior is non-trivial. | Makes failure triage faster and profile composition explicit. |
|
||||||
|
| Runtime policy | Never run `slow` tests in the default `unit` profile unless explicitly required. | Preserves turnaround for PR feedback while preserving deep checks. |
|
||||||
|
| Change risk | Any persistence or compatibility-affecting change must include `compat` in validation. | Protects long-lived binary artifact contracts. |
|
||||||
|
| Mutation resistance | `fuzz`/`property` sets should be gated to dedicated profiles. | Limits flakiness exposure and controls CI resource cost. |
|
||||||
|
|
||||||
|
## Suggested CI profiles
|
||||||
|
|
||||||
|
These are recommended launch profiles for local and CI usage and are also exposed as Gradle tasks:
|
||||||
|
|
||||||
|
- **Profile: `ci-smoke` (fast feedback):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=unit -DexcludeTags=slow
|
||||||
|
./gradlew ciSmoke
|
||||||
|
```
|
||||||
|
|
||||||
|
`ciSmoke` also excludes `org.egothor.stemmer.CompileIntegrationTest*` at test-name filter level as a
|
||||||
|
defensive fallback in case of future tag drift.
|
||||||
|
`ciRelease` also excludes
|
||||||
|
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` at filter level.
|
||||||
|
|
||||||
|
- **Profile: `ci-core` (core behavioral coverage):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=unit,trie,frequency-trie,property
|
||||||
|
./gradlew ciCore
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Profile: `ci-integration` (pipeline correctness):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=integration
|
||||||
|
./gradlew ciIntegration
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Profile: `ci-slow` (explicit heavy validation):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew ciSlow
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Profile: `ci-compat` (artifact stability):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=compat,regression
|
||||||
|
./gradlew ciCompat
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Profile: `ci-release` (strong confidence before release):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DexcludeTags=slow
|
||||||
|
./gradlew ciRelease
|
||||||
|
```
|
||||||
|
`ciRelease` is non-slow by policy and uses the same defensive name-based exclusion for
|
||||||
|
`org.egothor.stemmer.CompileIntegrationTest*` and
|
||||||
|
`org.egothor.stemmer.StemmerPatchTrieLoaderTest$BundledDictionaryTests*` in addition to tag filtering.
|
||||||
|
|
||||||
|
- **Profile: `ci-nightly` (extended hardening):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=fuzz
|
||||||
|
./gradlew ciNightly
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Profile: `ci` (enterprise umbrella):**
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew ci
|
||||||
|
```
|
||||||
|
|
||||||
|
`ci` and `ciRelease` intentionally do **not** include `slow` paths. Run `ciSlow` explicitly for production-dictionary stress and long-running corpus checks.
|
||||||
|
|
||||||
|
## Practical examples
|
||||||
|
|
||||||
|
All examples use Gradle with JUnit Platform integration:
|
||||||
|
|
||||||
|
- Only unit tests:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=unit
|
||||||
|
```
|
||||||
|
|
||||||
|
- Integration tests only:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=integration
|
||||||
|
```
|
||||||
|
|
||||||
|
- Only trie subsystem tests:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=trie
|
||||||
|
```
|
||||||
|
|
||||||
|
- Deterministic fuzz checks:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=fuzz
|
||||||
|
```
|
||||||
|
|
||||||
|
- Property tests:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=property
|
||||||
|
```
|
||||||
|
|
||||||
|
- Stemmer + patch command behavior:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=stemmer,patch
|
||||||
|
```
|
||||||
|
|
||||||
|
- Compatibility artifacts and regression checks:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=compat
|
||||||
|
```
|
||||||
|
|
||||||
|
- Keep regression suite and remove long-running cases:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=regression -DexcludeTags=slow
|
||||||
|
```
|
||||||
|
|
||||||
|
- Core + patch behavior:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=trie,patch
|
||||||
|
```
|
||||||
|
|
||||||
|
- Deterministic compatibility and persistence checks:
|
||||||
|
|
||||||
|
```
|
||||||
|
./gradlew test -DincludeTags=compat,determinism,serialization
|
||||||
|
```
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- `-DincludeTags` and `-DexcludeTags` are interpreted by Gradle task filtering and forwarded into
|
||||||
|
JUnit tag filtering.
|
||||||
|
- Class-name filtering is also available via Gradle test selectors where needed
|
||||||
|
(for example, `--tests *CompileTest`), but tag filtering remains the default
|
||||||
|
execution strategy.
|
||||||
|
- `-DincludeTags` supports comma-separated literal tags. When you need a single exact tag with special
|
||||||
|
characters, quote the argument for the shell.
|
||||||
@@ -84,7 +84,7 @@ publishing {
|
|||||||
}
|
}
|
||||||
|
|
||||||
signing {
|
signing {
|
||||||
required { !version.toString().endsWith('-SNAPSHOT') }
|
required = !version.toString().endsWith('-SNAPSHOT')
|
||||||
if (signingKey != null && !signingKey.isBlank()) {
|
if (signingKey != null && !signingKey.isBlank()) {
|
||||||
useInMemoryPgpKeys(signingKey, signingPassword)
|
useInMemoryPgpKeys(signingKey, signingPassword)
|
||||||
sign publishing.publications.mavenJava
|
sign publishing.publications.mavenJava
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ nav:
|
|||||||
- Overview: architecture-and-reduction.md
|
- Overview: architecture-and-reduction.md
|
||||||
- Architecture: architecture.md
|
- Architecture: architecture.md
|
||||||
- Reduction Semantics: reduction-semantics.md
|
- Reduction Semantics: reduction-semantics.md
|
||||||
|
- Lookup Edge Optimization: lookup-edge-optimization.md
|
||||||
- Compatibility and Guarantees: compatibility-and-guarantees.md
|
- Compatibility and Guarantees: compatibility-and-guarantees.md
|
||||||
|
|
||||||
- Dictionaries:
|
- Dictionaries:
|
||||||
@@ -63,3 +64,4 @@ nav:
|
|||||||
- Quality and Operations: quality-and-operations.md
|
- Quality and Operations: quality-and-operations.md
|
||||||
- Benchmarking: benchmarking.md
|
- Benchmarking: benchmarking.md
|
||||||
- Reports: reports.md
|
- Reports: reports.md
|
||||||
|
- Test taxonomy and execution filtering: test-taxonomy-and-filtering.md
|
||||||
|
|||||||
@@ -31,11 +31,13 @@
|
|||||||
package org.egothor.stemmer.benchmark;
|
package org.egothor.stemmer.benchmark;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import org.egothor.stemmer.FrequencyTrie;
|
import org.egothor.stemmer.FrequencyTrie;
|
||||||
import org.egothor.stemmer.PatchCommandEncoder;
|
import org.egothor.stemmer.PatchCommandEncoder;
|
||||||
import org.egothor.stemmer.ReductionMode;
|
import org.egothor.stemmer.ReductionMode;
|
||||||
import org.egothor.stemmer.ReductionSettings;
|
import org.egothor.stemmer.ReductionSettings;
|
||||||
|
import org.egothor.stemmer.ValueCount;
|
||||||
import org.openjdk.jmh.annotations.Benchmark;
|
import org.openjdk.jmh.annotations.Benchmark;
|
||||||
import org.openjdk.jmh.annotations.BenchmarkMode;
|
import org.openjdk.jmh.annotations.BenchmarkMode;
|
||||||
import org.openjdk.jmh.annotations.Level;
|
import org.openjdk.jmh.annotations.Level;
|
||||||
@@ -97,12 +99,45 @@ public class FrequencyTrieLookupBenchmark {
|
|||||||
*/
|
*/
|
||||||
private String[] lookupKeys;
|
private String[] lookupKeys;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lookup keys as normalized caller-owned character storage.
|
||||||
|
*/
|
||||||
|
private char[][] lookupKeyCharacters;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Keys that are known to return multiple patch candidates from
|
* Keys that are known to return multiple patch candidates from
|
||||||
* {@code getAll()}.
|
* {@code getAll()}.
|
||||||
*/
|
*/
|
||||||
private String[] ambiguousLookupKeys;
|
private String[] ambiguousLookupKeys;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ambiguous lookup keys as normalized caller-owned character storage.
|
||||||
|
*/
|
||||||
|
private char[][] ambiguousLookupKeyCharacters;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Preferred patches aligned with {@link #lookupKeys}.
|
||||||
|
*/
|
||||||
|
private String[] preferredPatches;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reusable output buffer for patch application benchmarks.
|
||||||
|
*/
|
||||||
|
private char[] outputBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mutable field consumed by visitor sinks.
|
||||||
|
*/
|
||||||
|
private int visitorAccumulator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sink used by visitor lookup benchmarks without per-invocation allocation.
|
||||||
|
*/
|
||||||
|
private final FrequencyTrie.EntrySink<String> visitorSink = (value, count, rank) -> {
|
||||||
|
this.visitorAccumulator += value.length() + count + rank;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initializes the benchmark state.
|
* Initializes the benchmark state.
|
||||||
*
|
*
|
||||||
@@ -116,6 +151,23 @@ public class FrequencyTrieLookupBenchmark {
|
|||||||
this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true);
|
this.trie = BenchmarkCorpusSupport.compilePatchTrie(corpus.dictionaryText(), settings, true);
|
||||||
this.lookupKeys = corpus.lookupKeys();
|
this.lookupKeys = corpus.lookupKeys();
|
||||||
this.ambiguousLookupKeys = corpus.ambiguousLookupKeys();
|
this.ambiguousLookupKeys = corpus.ambiguousLookupKeys();
|
||||||
|
this.lookupKeyCharacters = toCharArrays(this.lookupKeys);
|
||||||
|
this.ambiguousLookupKeyCharacters = toCharArrays(this.ambiguousLookupKeys);
|
||||||
|
this.preferredPatches = new String[this.lookupKeys.length];
|
||||||
|
int maxKeyLength = 0;
|
||||||
|
for (int index = 0; index < this.lookupKeys.length; index++) {
|
||||||
|
this.preferredPatches[index] = this.trie.get(this.lookupKeys[index]);
|
||||||
|
maxKeyLength = Math.max(maxKeyLength, this.lookupKeys[index].length());
|
||||||
|
}
|
||||||
|
this.outputBuffer = new char[maxKeyLength + 32];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static char[][] toCharArrays(final String[] values) {
|
||||||
|
final char[][] characters = new char[values.length][];
|
||||||
|
for (int index = 0; index < values.length; index++) {
|
||||||
|
characters[index] = values[index].toCharArray();
|
||||||
|
}
|
||||||
|
return characters;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,6 +207,61 @@ public class FrequencyTrieLookupBenchmark {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Measures retrieval of all patch candidates through caller-owned normalized
|
||||||
|
* character storage and a visitor sink.
|
||||||
|
*
|
||||||
|
* @param state prepared lookup state
|
||||||
|
* @param blackhole sink preventing dead-code elimination
|
||||||
|
*/
|
||||||
|
@Benchmark
|
||||||
|
public void lookupAllPatchesWithNormalizedCharVisitor(final LookupState state, final Blackhole blackhole) {
|
||||||
|
final char[][] keys = state.ambiguousLookupKeyCharacters;
|
||||||
|
for (char[] key : keys) {
|
||||||
|
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
|
||||||
|
if (count < 2) {
|
||||||
|
throw new IllegalStateException("Expected multiple patches for benchmark key.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
blackhole.consume(state.visitorAccumulator);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Measures counted candidate retrieval through the allocating entry API.
|
||||||
|
*
|
||||||
|
* @param state prepared lookup state
|
||||||
|
* @param blackhole sink preventing dead-code elimination
|
||||||
|
*/
|
||||||
|
@Benchmark
|
||||||
|
public void lookupPatchEntries(final LookupState state, final Blackhole blackhole) {
|
||||||
|
final String[] keys = state.ambiguousLookupKeys;
|
||||||
|
for (String key : keys) {
|
||||||
|
final List<ValueCount<String>> entries = state.trie.getEntries(key);
|
||||||
|
if (entries.size() < 2) {
|
||||||
|
throw new IllegalStateException("Expected multiple entries for key " + key + '.');
|
||||||
|
}
|
||||||
|
blackhole.consume(entries);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Measures counted candidate retrieval through the visitor API.
|
||||||
|
*
|
||||||
|
* @param state prepared lookup state
|
||||||
|
* @param blackhole sink preventing dead-code elimination
|
||||||
|
*/
|
||||||
|
@Benchmark
|
||||||
|
public void lookupPatchEntriesWithVisitor(final LookupState state, final Blackhole blackhole) {
|
||||||
|
final char[][] keys = state.ambiguousLookupKeyCharacters;
|
||||||
|
for (char[] key : keys) {
|
||||||
|
final int count = state.trie.getAllNormalized(key, 0, key.length, state.visitorSink, Integer.MAX_VALUE);
|
||||||
|
if (count < 2) {
|
||||||
|
throw new IllegalStateException("Expected multiple entries for benchmark key.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
blackhole.consume(state.visitorAccumulator);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Measures end-to-end preferred stemming from lookup plus patch application.
|
* Measures end-to-end preferred stemming from lookup plus patch application.
|
||||||
*
|
*
|
||||||
@@ -170,6 +277,28 @@ public class FrequencyTrieLookupBenchmark {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Measures patch application into caller-owned output storage.
|
||||||
|
*
|
||||||
|
* @param state prepared lookup state
|
||||||
|
* @param blackhole sink preventing dead-code elimination
|
||||||
|
*/
|
||||||
|
@Benchmark
|
||||||
|
public void applyPreferredPatchToBuffer(final LookupState state, final Blackhole blackhole) {
|
||||||
|
final String[] keys = state.lookupKeys;
|
||||||
|
final String[] patches = state.preferredPatches;
|
||||||
|
final char[] output = state.outputBuffer;
|
||||||
|
for (int index = 0; index < keys.length; index++) {
|
||||||
|
final int length = PatchCommandEncoder.applyTo(keys[index], patches[index],
|
||||||
|
state.trie.traversalDirection(), output, 0, output.length);
|
||||||
|
if (length == PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY) {
|
||||||
|
throw new IllegalStateException("Output buffer too small for key " + keys[index] + '.');
|
||||||
|
}
|
||||||
|
blackhole.consume(length);
|
||||||
|
blackhole.consume(output[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Measures end-to-end full candidate stemming from {@code getAll()} plus
|
* Measures end-to-end full candidate stemming from {@code getAll()} plus
|
||||||
* patch application.
|
* patch application.
|
||||||
|
|||||||
@@ -48,8 +48,7 @@ public enum CaseProcessingMode {
|
|||||||
AS_IS,
|
AS_IS,
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Normalizes all dictionary content to lower case using
|
* Normalizes all dictionary content to lower case using {@link Locale#ROOT}.
|
||||||
* {@link Locale#ROOT}.
|
|
||||||
*/
|
*/
|
||||||
LOWERCASE_WITH_LOCALE_ROOT
|
LOWERCASE_WITH_LOCALE_ROOT
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,12 +93,12 @@ final class DiacriticStripper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Removes supported diacritic marks and common Latin ligatures from the supplied
|
* Removes supported diacritic marks and common Latin ligatures from the
|
||||||
* text.
|
* supplied text.
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* The method returns the original {@link String} instance when no replacement is
|
* The method returns the original {@link String} instance when no replacement
|
||||||
* required, avoiding an unnecessary allocation on the common ASCII path.
|
* is required, avoiding an unnecessary allocation on the common ASCII path.
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* @param input text to normalize
|
* @param input text to normalize
|
||||||
|
|||||||
@@ -51,7 +51,6 @@ import java.util.logging.Logger;
|
|||||||
import org.egothor.stemmer.trie.CompiledNode;
|
import org.egothor.stemmer.trie.CompiledNode;
|
||||||
import org.egothor.stemmer.trie.LocalValueSummary;
|
import org.egothor.stemmer.trie.LocalValueSummary;
|
||||||
import org.egothor.stemmer.trie.MutableNode;
|
import org.egothor.stemmer.trie.MutableNode;
|
||||||
import org.egothor.stemmer.trie.NodeData;
|
|
||||||
import org.egothor.stemmer.trie.ReducedNode;
|
import org.egothor.stemmer.trie.ReducedNode;
|
||||||
import org.egothor.stemmer.trie.ReductionContext;
|
import org.egothor.stemmer.trie.ReductionContext;
|
||||||
import org.egothor.stemmer.trie.ReductionSignature;
|
import org.egothor.stemmer.trie.ReductionSignature;
|
||||||
@@ -87,7 +86,6 @@ import org.egothor.stemmer.trie.ReductionSignature;
|
|||||||
*
|
*
|
||||||
* @param <V> value type
|
* @param <V> value type
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
|
||||||
public final class FrequencyTrie<V> {
|
public final class FrequencyTrie<V> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -121,7 +119,8 @@ public final class FrequencyTrie<V> {
|
|||||||
private final boolean removeDiacritics;
|
private final boolean removeDiacritics;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shared empty array instance for empty lookup results from {@link #getAll(String)}.
|
* Shared empty array instance for empty lookup results from
|
||||||
|
* {@link #getAll(String)}.
|
||||||
*/
|
*/
|
||||||
private final V[] emptyValues;
|
private final V[] emptyValues;
|
||||||
|
|
||||||
@@ -130,11 +129,59 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
private static final int STREAM_MAGIC = 0x45475452;
|
private static final int STREAM_MAGIC = 0x45475452;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimum supported stream version constant retained for explicit range checks.
|
||||||
|
*/
|
||||||
|
private static final int MIN_STREAM_VERSION = 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of stored values for which {@link #getEntries(String)} can return an
|
||||||
|
* empty result.
|
||||||
|
*/
|
||||||
|
private static final int NO_VALUE_COUNT = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of stored values for which {@link #getEntries(String)} can use a
|
||||||
|
* one-item immutable list special case.
|
||||||
|
*/
|
||||||
|
private static final int SINGLE_VALUE_COUNT = 1;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Binary format version.
|
* Binary format version.
|
||||||
*/
|
*/
|
||||||
private static final int STREAM_VERSION = 5;
|
private static final int STREAM_VERSION = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Version where traversal-direction ordinal is persisted.
|
||||||
|
*/
|
||||||
|
private static final int TRAVERSAL_VERSION = 2;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Version where compact reduction metadata is persisted.
|
||||||
|
*/
|
||||||
|
private static final int REDUCTION_VERSION = 3;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Version where case-processing mode ordinal is persisted.
|
||||||
|
*/
|
||||||
|
private static final int CASE_VERSION = 4;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Argument name for lookup keys.
|
||||||
|
*/
|
||||||
|
private static final String ARG_KEY = "key";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default dense child lookup span in code points used when materializing
|
||||||
|
* compiled nodes without an explicit override.
|
||||||
|
* <p>
|
||||||
|
* Increasing this value increases the chance of direct array indexing for child
|
||||||
|
* lookup at runtime at the cost of per-node dense table memory for compact
|
||||||
|
* character spans.
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the current persisted binary stream format version.
|
* Returns the current persisted binary stream format version.
|
||||||
*
|
*
|
||||||
@@ -150,6 +197,30 @@ public final class FrequencyTrie<V> {
|
|||||||
return STREAM_VERSION;
|
return STREAM_VERSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Receives trie values during visitor-style lookup.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Implementations are caller-owned and are not retained by the trie. Returning
|
||||||
|
* {@code false} stops iteration after the current callback.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param <V> value type
|
||||||
|
*/
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface EntrySink<V> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Accepts one ordered local value.
|
||||||
|
*
|
||||||
|
* @param value stored value
|
||||||
|
* @param count stored local occurrence count
|
||||||
|
* @param rank zero-based rank in deterministic local ordering
|
||||||
|
* @return {@code true} to continue iteration, {@code false} to stop
|
||||||
|
*/
|
||||||
|
boolean accept(V value, int count, int rank);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a new compiled trie instance.
|
* Creates a new compiled trie instance.
|
||||||
*
|
*
|
||||||
@@ -188,7 +259,7 @@ public final class FrequencyTrie<V> {
|
|||||||
* @throws NullPointerException if {@code key} is {@code null}
|
* @throws NullPointerException if {@code key} is {@code null}
|
||||||
*/
|
*/
|
||||||
public V get(final String key) {
|
public V get(final String key) {
|
||||||
Objects.requireNonNull(key, "key");
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||||
if (node == null) {
|
if (node == null) {
|
||||||
return null;
|
return null;
|
||||||
@@ -225,7 +296,7 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||||
public V[] getAll(final String key) {
|
public V[] getAll(final String key) {
|
||||||
Objects.requireNonNull(key, "key");
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||||
if (node == null) {
|
if (node == null) {
|
||||||
return this.emptyValues;
|
return this.emptyValues;
|
||||||
@@ -259,9 +330,8 @@ public final class FrequencyTrie<V> {
|
|||||||
* if the key does not exist or no value is stored at the addressed node
|
* if the key does not exist or no value is stored at the addressed node
|
||||||
* @throws NullPointerException if {@code key} is {@code null}
|
* @throws NullPointerException if {@code key} is {@code null}
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
|
||||||
public List<ValueCount<V>> getEntries(final String key) {
|
public List<ValueCount<V>> getEntries(final String key) {
|
||||||
Objects.requireNonNull(key, "key");
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||||
if (node == null) {
|
if (node == null) {
|
||||||
return List.of();
|
return List.of();
|
||||||
@@ -269,11 +339,11 @@ public final class FrequencyTrie<V> {
|
|||||||
|
|
||||||
final V[] orderedValues = node.orderedValues();
|
final V[] orderedValues = node.orderedValues();
|
||||||
final int valueCount = orderedValues.length;
|
final int valueCount = orderedValues.length;
|
||||||
if (valueCount == 0) {
|
if (valueCount == NO_VALUE_COUNT) {
|
||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (valueCount == 1) {
|
if (valueCount == SINGLE_VALUE_COUNT) {
|
||||||
return List.of(new ValueCount<>(orderedValues[0], node.orderedCounts()[0]));
|
return List.of(new ValueCount<>(orderedValues[0], node.orderedCounts()[0]));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,6 +355,132 @@ public final class FrequencyTrie<V> {
|
|||||||
return Collections.unmodifiableList(entries);
|
return Collections.unmodifiableList(entries);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits all values stored at the node addressed by an already-normalized
|
||||||
|
* {@code char[]} key slice.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* This method bypasses {@link TrieMetadata#caseProcessingMode()} and
|
||||||
|
* {@link TrieMetadata#diacriticProcessingMode()}. The caller must provide input
|
||||||
|
* normalized exactly as required by this trie's metadata. The trie is immutable
|
||||||
|
* and thread-safe for concurrent reads; the supplied sink is caller-owned and
|
||||||
|
* is not retained.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param key normalized key storage
|
||||||
|
* @param offset first character offset
|
||||||
|
* @param length number of characters to read
|
||||||
|
* @param sink value sink
|
||||||
|
* @param maxResults maximum number of results to visit
|
||||||
|
* @return number of visited values
|
||||||
|
* @throws NullPointerException if {@code key} or {@code sink} is
|
||||||
|
* {@code null}
|
||||||
|
* @throws IndexOutOfBoundsException if the key slice is invalid
|
||||||
|
* @throws IllegalArgumentException if {@code maxResults} is negative
|
||||||
|
*/
|
||||||
|
public int getAllNormalized(final char[] key, final int offset, final int length, final EntrySink<? super V> sink,
|
||||||
|
final int maxResults) {
|
||||||
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
|
Objects.requireNonNull(sink, "sink");
|
||||||
|
Objects.checkFromIndexSize(offset, length, key.length);
|
||||||
|
validateMaxResults(maxResults);
|
||||||
|
if (maxResults == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return visitNode(findNode(key, offset, length), sink, maxResults);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits all values stored at the node addressed by an already-normalized
|
||||||
|
* character sequence.
|
||||||
|
*
|
||||||
|
* @param key normalized key
|
||||||
|
* @param sink value sink
|
||||||
|
* @param maxResults maximum number of results to visit
|
||||||
|
* @return number of visited values
|
||||||
|
* @throws NullPointerException if {@code key} or {@code sink} is
|
||||||
|
* {@code null}
|
||||||
|
* @throws IllegalArgumentException if {@code maxResults} is negative
|
||||||
|
* @see #getAllNormalized(char[], int, int, EntrySink, int)
|
||||||
|
*/
|
||||||
|
public int getAllNormalized(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
|
||||||
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
|
Objects.requireNonNull(sink, "sink");
|
||||||
|
validateMaxResults(maxResults);
|
||||||
|
if (maxResults == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return visitNode(findNode(key), sink, maxResults);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits the first value stored at the node addressed by an already-normalized
|
||||||
|
* {@code char[]} key slice.
|
||||||
|
*
|
||||||
|
* @param key normalized key storage
|
||||||
|
* @param offset first character offset
|
||||||
|
* @param length number of characters to read
|
||||||
|
* @param sink value sink
|
||||||
|
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||||
|
* @see #getAllNormalized(char[], int, int, EntrySink, int)
|
||||||
|
*/
|
||||||
|
public boolean getFirstNormalized(final char[] key, final int offset, final int length,
|
||||||
|
final EntrySink<? super V> sink) {
|
||||||
|
return getAllNormalized(key, offset, length, sink, 1) == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits the first value stored at the node addressed by an already-normalized
|
||||||
|
* character sequence.
|
||||||
|
*
|
||||||
|
* @param key normalized key
|
||||||
|
* @param sink value sink
|
||||||
|
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||||
|
* @see #getAllNormalized(CharSequence, EntrySink, int)
|
||||||
|
*/
|
||||||
|
public boolean getFirstNormalized(final CharSequence key, final EntrySink<? super V> sink) {
|
||||||
|
return getAllNormalized(key, sink, 1) == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits all values stored at the node addressed by the supplied key, applying
|
||||||
|
* metadata-driven lookup normalization when required.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* This method preserves the same lookup normalization semantics as
|
||||||
|
* {@link #getAll(String)}. It may allocate when metadata requires lowercase or
|
||||||
|
* diacritic normalization.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param key key to resolve
|
||||||
|
* @param sink value sink
|
||||||
|
* @param maxResults maximum number of results to visit
|
||||||
|
* @return number of visited values
|
||||||
|
*/
|
||||||
|
public int getAll(final CharSequence key, final EntrySink<? super V> sink, final int maxResults) {
|
||||||
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
|
Objects.requireNonNull(sink, "sink");
|
||||||
|
validateMaxResults(maxResults);
|
||||||
|
if (maxResults == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
final CharSequence normalized = normalizeLookupKey(key);
|
||||||
|
return visitNode(findNode(normalized), sink, maxResults);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits the first value stored at the node addressed by the supplied key,
|
||||||
|
* applying metadata-driven lookup normalization when required.
|
||||||
|
*
|
||||||
|
* @param key key to resolve
|
||||||
|
* @param sink value sink
|
||||||
|
* @return {@code true} when a value was visited, otherwise {@code false}
|
||||||
|
* @see #getAll(CharSequence, EntrySink, int)
|
||||||
|
*/
|
||||||
|
public boolean getFirst(final CharSequence key, final EntrySink<? super V> sink) {
|
||||||
|
return getAll(key, sink, 1) == 1;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the logical key traversal direction used by this trie.
|
* Returns the logical key traversal direction used by this trie.
|
||||||
*
|
*
|
||||||
@@ -383,47 +579,32 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||||
final ValueStreamCodec<V> valueCodec) throws IOException {
|
final ValueStreamCodec<V> valueCodec) throws IOException {
|
||||||
Objects.requireNonNull(inputStream, "inputStream");
|
return readFrom(inputStream, arrayFactory, valueCodec, -1);
|
||||||
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
}
|
||||||
Objects.requireNonNull(valueCodec, "valueCodec");
|
|
||||||
|
|
||||||
final DataInputStream dataInput; // NOPMD
|
/**
|
||||||
if (inputStream instanceof DataInputStream) {
|
* Reads a compiled trie from the supplied input stream, optionally overriding
|
||||||
dataInput = (DataInputStream) inputStream;
|
* dense child-index span configuration.
|
||||||
} else {
|
* <p>
|
||||||
dataInput = new DataInputStream(inputStream);
|
* This setting is applied only while materializing the in-memory compiled
|
||||||
}
|
* representation during load. It is not serialized in {@link TrieMetadata}, so
|
||||||
|
* each load can independently choose its own runtime lookup trade-off.
|
||||||
final int magic = dataInput.readInt();
|
* </p>
|
||||||
if (magic != STREAM_MAGIC) {
|
*
|
||||||
throw new IOException("Unsupported trie stream header: " + Integer.toHexString(magic));
|
* @param inputStream source input stream
|
||||||
}
|
* @param arrayFactory array factory used to create typed arrays
|
||||||
|
* @param valueCodec codec used to read values
|
||||||
final int version = dataInput.readInt();
|
* @param maxExpandedIndex dense lookup span override; zero disables dense
|
||||||
if (version != 1 && version != 3 && version != 4 && version != STREAM_VERSION) {
|
* lookup, negative values use
|
||||||
throw new IOException("Unsupported trie stream version: " + version);
|
* {@link #DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
}
|
* @param <V> value type
|
||||||
|
* @return deserialized compiled trie
|
||||||
final int nodeCount = dataInput.readInt();
|
* @throws NullPointerException if any argument is {@code null}
|
||||||
if (nodeCount < 0) {
|
* @throws IOException if reading fails or the binary format is invalid
|
||||||
throw new IOException("Negative node count: " + nodeCount);
|
*/
|
||||||
}
|
public static <V> FrequencyTrie<V> readFrom(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||||
|
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
||||||
final int rootNodeId = dataInput.readInt();
|
return CompiledTrieReader.read(inputStream, arrayFactory, valueCodec, maxExpandedIndex);
|
||||||
if (rootNodeId < 0 || rootNodeId >= nodeCount) {
|
|
||||||
throw new IOException("Invalid root node id: " + rootNodeId);
|
|
||||||
}
|
|
||||||
|
|
||||||
final TrieMetadata metadata = readMetadata(dataInput, version);
|
|
||||||
|
|
||||||
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount);
|
|
||||||
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
|
||||||
|
|
||||||
if (LOGGER.isLoggable(Level.FINE)) {
|
|
||||||
LOGGER.log(Level.FINE, "Read compiled trie with {0} canonical nodes.", nodeCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
return new FrequencyTrie<>(arrayFactory, rootNode, metadata);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -438,73 +619,6 @@ public final class FrequencyTrie<V> {
|
|||||||
dataOutput.writeUTF(metadata.toTextBlock());
|
dataOutput.writeUTF(metadata.toTextBlock());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads persisted trie metadata while remaining backward compatible with
|
|
||||||
* earlier stream versions.
|
|
||||||
*
|
|
||||||
* @param dataInput input stream
|
|
||||||
* @param version persisted stream version
|
|
||||||
* @return deserialized metadata
|
|
||||||
* @throws IOException if the metadata section is invalid
|
|
||||||
*/
|
|
||||||
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version) throws IOException {
|
|
||||||
if (version >= 5) { // NOPMD
|
|
||||||
try {
|
|
||||||
return TrieMetadata.fromTextBlock(version, dataInput.readUTF());
|
|
||||||
} catch (IllegalArgumentException exception) {
|
|
||||||
throw new IOException("Invalid metadata block.", exception);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final WordTraversalDirection traversalDirection;
|
|
||||||
if (version >= 2) { // NOPMD
|
|
||||||
final int traversalDirectionOrdinal = dataInput.readInt();
|
|
||||||
final WordTraversalDirection[] traversalDirections = WordTraversalDirection.values();
|
|
||||||
if (traversalDirectionOrdinal < 0 || traversalDirectionOrdinal >= traversalDirections.length) {
|
|
||||||
throw new IOException("Invalid traversal direction ordinal: " + traversalDirectionOrdinal);
|
|
||||||
}
|
|
||||||
traversalDirection = traversalDirections[traversalDirectionOrdinal];
|
|
||||||
} else {
|
|
||||||
traversalDirection = WordTraversalDirection.BACKWARD;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (version < 3) { // NOPMD
|
|
||||||
return TrieMetadata.legacy(version, traversalDirection);
|
|
||||||
}
|
|
||||||
|
|
||||||
final ReductionMode[] reductionModes = ReductionMode.values();
|
|
||||||
final int reductionModeOrdinal = dataInput.readInt();
|
|
||||||
if (reductionModeOrdinal < 0 || reductionModeOrdinal >= reductionModes.length) {
|
|
||||||
throw new IOException("Invalid reduction mode ordinal: " + reductionModeOrdinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
final int dominantWinnerMinPercent = dataInput.readInt();
|
|
||||||
final int dominantWinnerOverSecondRatio = dataInput.readInt(); // NOPMD
|
|
||||||
|
|
||||||
final DiacriticProcessingMode[] diacriticProcessingModes = DiacriticProcessingMode.values();
|
|
||||||
final int diacriticProcessingModeOrdinal = dataInput.readInt(); // NOPMD
|
|
||||||
if (diacriticProcessingModeOrdinal < 0 || diacriticProcessingModeOrdinal >= diacriticProcessingModes.length) {
|
|
||||||
throw new IOException("Invalid diacritic processing mode ordinal: " + diacriticProcessingModeOrdinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
final CaseProcessingMode caseProcessingMode;
|
|
||||||
if (version >= 4) { // NOPMD
|
|
||||||
final CaseProcessingMode[] caseProcessingModes = CaseProcessingMode.values();
|
|
||||||
final int caseProcessingModeOrdinal = dataInput.readInt();
|
|
||||||
if (caseProcessingModeOrdinal < 0 || caseProcessingModeOrdinal >= caseProcessingModes.length) {
|
|
||||||
throw new IOException("Invalid case processing mode ordinal: " + caseProcessingModeOrdinal);
|
|
||||||
}
|
|
||||||
caseProcessingMode = caseProcessingModes[caseProcessingModeOrdinal];
|
|
||||||
} else {
|
|
||||||
caseProcessingMode = CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
|
||||||
}
|
|
||||||
|
|
||||||
return new TrieMetadata(version, traversalDirection,
|
|
||||||
new ReductionSettings(reductionModes[reductionModeOrdinal], dominantWinnerMinPercent,
|
|
||||||
dominantWinnerOverSecondRatio),
|
|
||||||
diacriticProcessingModes[diacriticProcessingModeOrdinal], caseProcessingMode);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the number of canonical compiled nodes reachable from the root.
|
* Returns the number of canonical compiled nodes reachable from the root.
|
||||||
*
|
*
|
||||||
@@ -574,103 +688,222 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads all compiled nodes and resolves child references.
|
* Internal helper that materializes serialized trie data.
|
||||||
*
|
|
||||||
* @param dataInput input
|
|
||||||
* @param arrayFactory array factory
|
|
||||||
* @param valueCodec value codec
|
|
||||||
* @param nodeCount number of nodes
|
|
||||||
* @param <V> value type
|
|
||||||
* @return array of nodes indexed by serialized node identifier
|
|
||||||
* @throws IOException if reading fails or the stream is invalid
|
|
||||||
*/
|
|
||||||
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
|
|
||||||
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput, final IntFunction<V[]> arrayFactory,
|
|
||||||
final ValueStreamCodec<V> valueCodec, final int nodeCount) throws IOException {
|
|
||||||
final List<NodeData<V>> nodeDataList = new ArrayList<>(nodeCount);
|
|
||||||
|
|
||||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
|
||||||
final int edgeCount = dataInput.readInt();
|
|
||||||
if (edgeCount < 0) {
|
|
||||||
throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
final char[] edgeLabels = new char[edgeCount];
|
|
||||||
final int[] childNodeIds = new int[edgeCount];
|
|
||||||
|
|
||||||
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
|
||||||
edgeLabels[edgeIndex] = dataInput.readChar();
|
|
||||||
childNodeIds[edgeIndex] = dataInput.readInt();
|
|
||||||
}
|
|
||||||
|
|
||||||
validateSerializedEdges(nodeIndex, edgeLabels);
|
|
||||||
|
|
||||||
final int valueCount = dataInput.readInt();
|
|
||||||
if (valueCount < 0) {
|
|
||||||
throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
final V[] orderedValues = arrayFactory.apply(valueCount);
|
|
||||||
final int[] orderedCounts = new int[valueCount];
|
|
||||||
|
|
||||||
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
|
|
||||||
orderedValues[valueIndex] = valueCodec.read(dataInput);
|
|
||||||
orderedCounts[valueIndex] = dataInput.readInt();
|
|
||||||
if (orderedCounts[valueIndex] <= 0) {
|
|
||||||
throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index "
|
|
||||||
+ valueIndex + ": " + orderedCounts[valueIndex]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
nodeDataList.add(new NodeData<>(edgeLabels, childNodeIds, orderedValues, orderedCounts));
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
final CompiledNode<V>[] nodes = new CompiledNode[nodeCount];
|
|
||||||
|
|
||||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
|
||||||
final NodeData<V> nodeData = nodeDataList.get(nodeIndex);
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
final CompiledNode<V>[] children = new CompiledNode[nodeData.childNodeIds().length];
|
|
||||||
nodes[nodeIndex] = new CompiledNode<>(nodeData.edgeLabels(), children, nodeData.orderedValues(),
|
|
||||||
nodeData.orderedCounts());
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
|
||||||
final NodeData<V> nodeData = nodeDataList.get(nodeIndex);
|
|
||||||
final CompiledNode<V> node = nodes[nodeIndex];
|
|
||||||
|
|
||||||
for (int edgeIndex = 0; edgeIndex < nodeData.childNodeIds().length; edgeIndex++) {
|
|
||||||
final int childNodeId = nodeData.childNodeIds()[edgeIndex];
|
|
||||||
if (childNodeId < 0 || childNodeId >= nodeCount) {
|
|
||||||
throw new IOException("Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex
|
|
||||||
+ ": " + childNodeId);
|
|
||||||
}
|
|
||||||
node.children()[edgeIndex] = nodes[childNodeId];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nodes;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Validates the serialized edge-label sequence for one node.
|
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
* Compiled nodes rely on binary search for child lookup and therefore require
|
* Moving reader complexity into this helper keeps the public-facing class from
|
||||||
* edge labels to be stored in strict ascending order without duplicates.
|
* accumulating excessive class-level cyclomatic complexity while preserving the
|
||||||
* Rejecting malformed streams here keeps lookup semantics deterministic and
|
* same binary compatibility contract.
|
||||||
* avoids silently constructing a trie whose search behavior would be undefined.
|
* </p>
|
||||||
*
|
|
||||||
* @param nodeIndex serialized node identifier
|
|
||||||
* @param edgeLabels serialized edge labels
|
|
||||||
* @throws IOException if the edge labels are not strictly ascending
|
|
||||||
*/
|
*/
|
||||||
private static void validateSerializedEdges(final int nodeIndex, final char... edgeLabels) throws IOException {
|
private static final class CompiledTrieReader {
|
||||||
for (int edgeIndex = 1; edgeIndex < edgeLabels.length; edgeIndex++) {
|
|
||||||
if (edgeLabels[edgeIndex - 1] >= edgeLabels[edgeIndex]) {
|
private static <V> FrequencyTrie<V> read(final InputStream inputStream, final IntFunction<V[]> arrayFactory,
|
||||||
throw new IOException("Edge labels must be strictly ascending at node " + nodeIndex + ", edge index "
|
final ValueStreamCodec<V> valueCodec, final int maxExpandedIndex) throws IOException {
|
||||||
+ edgeIndex + ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
|
Objects.requireNonNull(inputStream, "inputStream");
|
||||||
|
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||||
|
Objects.requireNonNull(valueCodec, "valueCodec");
|
||||||
|
if (maxExpandedIndex < -1) {
|
||||||
|
throw new IllegalArgumentException("maxExpandedIndex must be >= -1.");
|
||||||
|
}
|
||||||
|
|
||||||
|
final DataInputStream dataInput = wrapInputStream(inputStream);
|
||||||
|
final int magic = dataInput.readInt();
|
||||||
|
if (magic != STREAM_MAGIC) {
|
||||||
|
throw new IOException("Unsupported trie stream header: " + Integer.toHexString(magic));
|
||||||
|
}
|
||||||
|
|
||||||
|
final int version = dataInput.readInt();
|
||||||
|
if (version < MIN_STREAM_VERSION || version > STREAM_VERSION) {
|
||||||
|
throw new IOException("Unsupported trie stream version: " + version);
|
||||||
|
}
|
||||||
|
|
||||||
|
final int nodeCount = dataInput.readInt();
|
||||||
|
if (nodeCount < 0) {
|
||||||
|
throw new IOException("Negative node count: " + nodeCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
final int rootNodeId = dataInput.readInt();
|
||||||
|
if (rootNodeId < 0 || rootNodeId >= nodeCount) {
|
||||||
|
throw new IOException("Invalid root node id: " + rootNodeId);
|
||||||
|
}
|
||||||
|
|
||||||
|
final TrieMetadata sourceMetadata = readMetadata(dataInput, version);
|
||||||
|
final int effectiveMaxExpandedIndex = maxExpandedIndex >= 0 ? maxExpandedIndex : DEFAULT_MAX_EXPANDED_INDEX;
|
||||||
|
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount,
|
||||||
|
effectiveMaxExpandedIndex);
|
||||||
|
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
||||||
|
|
||||||
|
if (LOGGER.isLoggable(Level.FINE)) {
|
||||||
|
LOGGER.log(Level.FINE, "Read compiled trie with {0} canonical nodes.", nodeCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new FrequencyTrie<>(arrayFactory, rootNode, sourceMetadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DataInputStream wrapInputStream(final InputStream inputStream) {
|
||||||
|
return inputStream instanceof DataInputStream ? (DataInputStream) inputStream
|
||||||
|
: new DataInputStream(inputStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version)
|
||||||
|
throws IOException {
|
||||||
|
if (version == STREAM_VERSION) {
|
||||||
|
return readTextMetadata(dataInput);
|
||||||
|
}
|
||||||
|
|
||||||
|
final WordTraversalDirection traversalDirection = readTraversalDirection(dataInput, version);
|
||||||
|
if (version < REDUCTION_VERSION) {
|
||||||
|
return TrieMetadata.legacy(version, traversalDirection);
|
||||||
|
}
|
||||||
|
|
||||||
|
final ReductionSettings reductionSettings = readReductionSettings(dataInput);
|
||||||
|
final DiacriticProcessingMode diacriticProcessingMode = readEnumByOrdinal(dataInput,
|
||||||
|
DiacriticProcessingMode.values(), "diacritic processing mode");
|
||||||
|
final CaseProcessingMode caseProcessingMode = version >= CASE_VERSION ? readCaseProcessingMode(dataInput)
|
||||||
|
: CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||||
|
return new TrieMetadata(version, traversalDirection, reductionSettings, diacriticProcessingMode,
|
||||||
|
caseProcessingMode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static TrieMetadata readTextMetadata(final DataInputStream dataInput) throws IOException {
|
||||||
|
try {
|
||||||
|
return TrieMetadata.fromTextBlock(STREAM_VERSION, dataInput.readUTF());
|
||||||
|
} catch (IllegalArgumentException exception) {
|
||||||
|
throw new IOException("Invalid metadata block.", exception);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static WordTraversalDirection readTraversalDirection(final DataInputStream dataInput, final int version)
|
||||||
|
throws IOException {
|
||||||
|
if (version < TRAVERSAL_VERSION) {
|
||||||
|
return WordTraversalDirection.BACKWARD;
|
||||||
|
}
|
||||||
|
return readEnumByOrdinal(dataInput, WordTraversalDirection.values(), "traversal direction");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ReductionSettings readReductionSettings(final DataInputStream dataInput) throws IOException {
|
||||||
|
final ReductionMode reductionMode = readEnumByOrdinal(dataInput, ReductionMode.values(), "reduction mode");
|
||||||
|
final int dominantWinnerMinPercent = dataInput.readInt();
|
||||||
|
final int dominantWinnerOverSecondRatio = dataInput.readInt(); // NOPMD
|
||||||
|
return new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CaseProcessingMode readCaseProcessingMode(final DataInputStream dataInput) throws IOException {
|
||||||
|
return readEnumByOrdinal(dataInput, CaseProcessingMode.values(), "case processing mode");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <E extends Enum<E>> E readEnumByOrdinal(final DataInputStream dataInput, final E[] values,
|
||||||
|
final String name) throws IOException {
|
||||||
|
final int ordinal = dataInput.readInt();
|
||||||
|
if (ordinal < 0 || ordinal >= values.length) {
|
||||||
|
throw new IOException("Invalid " + name + " ordinal: " + ordinal);
|
||||||
|
}
|
||||||
|
return values[ordinal];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <V> CompiledNode<V>[] readNodes(final DataInputStream dataInput,
|
||||||
|
final IntFunction<V[]> arrayFactory, final ValueStreamCodec<V> valueCodec, final int nodeCount,
|
||||||
|
final int maxExpandedIndex) throws IOException {
|
||||||
|
final char[][] edgeLabelsByNode = new char[nodeCount][];
|
||||||
|
final int[][] childNodeIdsByNode = new int[nodeCount][];
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
final V[][] orderedValuesByNode = (V[][]) new Object[nodeCount][];
|
||||||
|
final int[][] orderedCountsByNode = new int[nodeCount][];
|
||||||
|
|
||||||
|
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||||
|
final int edgeCount = dataInput.readInt();
|
||||||
|
if (edgeCount < 0) {
|
||||||
|
throw new IOException("Negative edge count at node " + nodeIndex + ": " + edgeCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
edgeLabelsByNode[nodeIndex] = new char[edgeCount];
|
||||||
|
childNodeIdsByNode[nodeIndex] = new int[edgeCount];
|
||||||
|
|
||||||
|
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
||||||
|
edgeLabelsByNode[nodeIndex][edgeIndex] = dataInput.readChar();
|
||||||
|
childNodeIdsByNode[nodeIndex][edgeIndex] = dataInput.readInt();
|
||||||
|
}
|
||||||
|
|
||||||
|
validateSerializedEdges(nodeIndex, edgeLabelsByNode[nodeIndex]);
|
||||||
|
|
||||||
|
final int valueCount = dataInput.readInt();
|
||||||
|
if (valueCount < 0) {
|
||||||
|
throw new IOException("Negative value count at node " + nodeIndex + ": " + valueCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
orderedValuesByNode[nodeIndex] = arrayFactory.apply(valueCount);
|
||||||
|
orderedCountsByNode[nodeIndex] = new int[valueCount];
|
||||||
|
|
||||||
|
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
|
||||||
|
orderedValuesByNode[nodeIndex][valueIndex] = valueCodec.read(dataInput);
|
||||||
|
orderedCountsByNode[nodeIndex][valueIndex] = dataInput.readInt();
|
||||||
|
if (orderedCountsByNode[nodeIndex][valueIndex] <= 0) {
|
||||||
|
throw new IOException("Non-positive stored count at node " + nodeIndex + ", value index "
|
||||||
|
+ valueIndex + ": " + orderedCountsByNode[nodeIndex][valueIndex]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
final CompiledNode<V>[] nodes = new CompiledNode[nodeCount];
|
||||||
|
final boolean[] inProgress = new boolean[nodeCount];
|
||||||
|
|
||||||
|
for (int nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) {
|
||||||
|
nodes[nodeIndex] = resolveNode(nodeIndex, edgeLabelsByNode, childNodeIdsByNode, orderedValuesByNode,
|
||||||
|
orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return nodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <V> CompiledNode<V> resolveNode(final int nodeIndex, final char[][] edgeLabelsByNode,
|
||||||
|
final int[][] childNodeIdsByNode, final V[][] orderedValuesByNode, final int[][] orderedCountsByNode,
|
||||||
|
final CompiledNode<V>[] nodes, final boolean[] inProgress, final int maxExpandedIndex)
|
||||||
|
throws IOException {
|
||||||
|
final CompiledNode<V> cachedNode = nodes[nodeIndex];
|
||||||
|
if (cachedNode != null) {
|
||||||
|
return cachedNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (inProgress[nodeIndex]) {
|
||||||
|
throw new IOException(
|
||||||
|
"Invalid serialized node graph: cyclic reference detected at node " + nodeIndex + '.');
|
||||||
|
}
|
||||||
|
inProgress[nodeIndex] = true;
|
||||||
|
try {
|
||||||
|
final char[] edgeLabels = edgeLabelsByNode[nodeIndex];
|
||||||
|
final int[] childNodeIds = childNodeIdsByNode[nodeIndex];
|
||||||
|
final int edgeCount = childNodeIds.length;
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
final CompiledNode<V>[] children = new CompiledNode[edgeCount];
|
||||||
|
|
||||||
|
for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
|
||||||
|
final int childNodeId = childNodeIds[edgeIndex];
|
||||||
|
if (childNodeId < 0 || childNodeId >= edgeLabelsByNode.length) {
|
||||||
|
throw new IOException("Invalid child node id at node " + nodeIndex + ", edge index " + edgeIndex
|
||||||
|
+ ": " + childNodeId);
|
||||||
|
}
|
||||||
|
children[edgeIndex] = resolveNode(childNodeId, edgeLabelsByNode, childNodeIdsByNode,
|
||||||
|
orderedValuesByNode, orderedCountsByNode, nodes, inProgress, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
final CompiledNode<V> node = new CompiledNode<>(edgeLabels, children, orderedValuesByNode[nodeIndex],
|
||||||
|
maxExpandedIndex, orderedCountsByNode[nodeIndex]);
|
||||||
|
nodes[nodeIndex] = node;
|
||||||
|
return node;
|
||||||
|
} finally {
|
||||||
|
inProgress[nodeIndex] = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void validateSerializedEdges(final int nodeIndex, final char... edgeLabels) throws IOException {
|
||||||
|
for (int edgeIndex = 1; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||||
|
if (edgeLabels[edgeIndex - 1] >= edgeLabels[edgeIndex]) {
|
||||||
|
throw new IOException(
|
||||||
|
"Edge labels must be strictly ascending at node " + nodeIndex + ", edge index " + edgeIndex
|
||||||
|
+ ": '" + edgeLabels[edgeIndex - 1] + "' then '" + edgeLabels[edgeIndex] + "'.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -682,6 +915,16 @@ public final class FrequencyTrie<V> {
|
|||||||
* @return compiled node, or {@code null} if the path does not exist
|
* @return compiled node, or {@code null} if the path does not exist
|
||||||
*/
|
*/
|
||||||
private CompiledNode<V> findNode(final String key) {
|
private CompiledNode<V> findNode(final String key) {
|
||||||
|
return findNode((CharSequence) key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Locates the compiled node for the supplied key.
|
||||||
|
*
|
||||||
|
* @param key already-normalized key to resolve
|
||||||
|
* @return compiled node, or {@code null} if the path does not exist
|
||||||
|
*/
|
||||||
|
private CompiledNode<V> findNode(final CharSequence key) {
|
||||||
CompiledNode<V> current = this.root;
|
CompiledNode<V> current = this.root;
|
||||||
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
||||||
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
|
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
|
||||||
@@ -702,6 +945,77 @@ public final class FrequencyTrie<V> {
|
|||||||
return current;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Locates the compiled node for the supplied key slice.
|
||||||
|
*
|
||||||
|
* @param key already-normalized key storage
|
||||||
|
* @param offset first character offset
|
||||||
|
* @param length number of characters to read
|
||||||
|
* @return compiled node, or {@code null} if the path does not exist
|
||||||
|
*/
|
||||||
|
private CompiledNode<V> findNode(final char[] key, final int offset, final int length) {
|
||||||
|
CompiledNode<V> current = this.root;
|
||||||
|
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
||||||
|
for (int traversalOffset = offset + length - 1; traversalOffset >= offset; traversalOffset--) {
|
||||||
|
current = current.findChild(key[traversalOffset]);
|
||||||
|
if (current == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int endExclusive = offset + length;
|
||||||
|
for (int traversalOffset = offset; traversalOffset < endExclusive; traversalOffset++) {
|
||||||
|
current = current.findChild(key[traversalOffset]);
|
||||||
|
if (current == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visits node-local values without allocating result containers.
|
||||||
|
*
|
||||||
|
* @param node resolved node, or {@code null}
|
||||||
|
* @param sink value sink
|
||||||
|
* @param maxResults maximum values to visit
|
||||||
|
* @return number of visited values
|
||||||
|
*/
|
||||||
|
private int visitNode(final CompiledNode<V> node, final EntrySink<? super V> sink, final int maxResults) {
|
||||||
|
if (node == null) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
final V[] orderedValues = node.orderedValues();
|
||||||
|
final int valueCount = Math.min(orderedValues.length, maxResults);
|
||||||
|
if (valueCount == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int[] orderedCounts = node.orderedCounts();
|
||||||
|
int visited = 0;
|
||||||
|
for (int rank = 0; rank < valueCount; rank++) {
|
||||||
|
visited++;
|
||||||
|
if (!sink.accept(orderedValues[rank], orderedCounts[rank], rank)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return visited;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates visitor maximum result count.
|
||||||
|
*
|
||||||
|
* @param maxResults maximum result count
|
||||||
|
*/
|
||||||
|
private static void validateMaxResults(final int maxResults) {
|
||||||
|
if (maxResults < 0) {
|
||||||
|
throw new IllegalArgumentException("maxResults must be non-negative.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Applies lookup-time case normalization according to persisted metadata.
|
* Applies lookup-time case normalization according to persisted metadata.
|
||||||
*
|
*
|
||||||
@@ -709,11 +1023,21 @@ public final class FrequencyTrie<V> {
|
|||||||
* @return normalized key for trie traversal
|
* @return normalized key for trie traversal
|
||||||
*/
|
*/
|
||||||
private String normalizeLookupKey(final String key) {
|
private String normalizeLookupKey(final String key) {
|
||||||
|
return normalizeLookupKey((CharSequence) key).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies lookup-time normalization according to persisted metadata.
|
||||||
|
*
|
||||||
|
* @param key lookup key
|
||||||
|
* @return normalized key for trie traversal
|
||||||
|
*/
|
||||||
|
private CharSequence normalizeLookupKey(final CharSequence key) {
|
||||||
if (!this.lowercasesLookupKeys && !this.removeDiacritics) {
|
if (!this.lowercasesLookupKeys && !this.removeDiacritics) {
|
||||||
return key;
|
return key;
|
||||||
}
|
}
|
||||||
|
|
||||||
String normalized = key;
|
String normalized = key.toString();
|
||||||
if (this.lowercasesLookupKeys) {
|
if (this.lowercasesLookupKeys) {
|
||||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
@@ -771,6 +1095,16 @@ public final class FrequencyTrie<V> {
|
|||||||
*/
|
*/
|
||||||
private final DiacriticProcessingMode diacriticProcessingMode;
|
private final DiacriticProcessingMode diacriticProcessingMode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dense edge lookup span threshold.
|
||||||
|
* <p>
|
||||||
|
* This value controls a speed/memory trade-off during freezing: dense child
|
||||||
|
* lookup tables are allocated only for nodes whose child labels fit in this
|
||||||
|
* span.
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
private final int maxExpandedIndex;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mutable root node.
|
* Mutable root node.
|
||||||
*/
|
*/
|
||||||
@@ -837,11 +1171,39 @@ public final class FrequencyTrie<V> {
|
|||||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
||||||
final DiacriticProcessingMode diacriticProcessingMode) {
|
final DiacriticProcessingMode diacriticProcessingMode) {
|
||||||
|
this(arrayFactory, reductionSettings, traversalDirection, caseProcessingMode, diacriticProcessingMode,
|
||||||
|
CompiledNode.DEFAULT_MAX_EXPANDED_INDEX);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new builder with the provided settings, explicit traversal
|
||||||
|
* direction, explicit case processing mode, explicit diacritic processing mode,
|
||||||
|
* and an explicit dense child lookup threshold.
|
||||||
|
*
|
||||||
|
* @param arrayFactory array factory
|
||||||
|
* @param reductionSettings reduction configuration
|
||||||
|
* @param traversalDirection logical key traversal direction
|
||||||
|
* @param caseProcessingMode dictionary case processing mode
|
||||||
|
* @param diacriticProcessingMode dictionary diacritic processing mode
|
||||||
|
* @param maxExpandedIndex dense lookup span override; zero disables
|
||||||
|
* dense lookup. Larger values increase direct
|
||||||
|
* indexing opportunities while potentially
|
||||||
|
* increasing materialization memory in nodes
|
||||||
|
* whose edge label span is within the limit.
|
||||||
|
* @throws NullPointerException if any argument is {@code null}
|
||||||
|
*/
|
||||||
|
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||||
|
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
||||||
|
final DiacriticProcessingMode diacriticProcessingMode, final int maxExpandedIndex) {
|
||||||
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||||
this.reductionSettings = Objects.requireNonNull(reductionSettings, "reductionSettings");
|
this.reductionSettings = Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||||
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||||
this.caseProcessingMode = Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
this.caseProcessingMode = Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||||
this.diacriticProcessingMode = Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
this.diacriticProcessingMode = Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||||
|
if (maxExpandedIndex < 0) {
|
||||||
|
throw new IllegalArgumentException("maxExpandedIndex must be non-negative.");
|
||||||
|
}
|
||||||
|
this.maxExpandedIndex = maxExpandedIndex;
|
||||||
this.root = new MutableNode<>();
|
this.root = new MutableNode<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -942,7 +1304,7 @@ public final class FrequencyTrie<V> {
|
|||||||
* @throws IllegalArgumentException if {@code count} is less than {@code 1}
|
* @throws IllegalArgumentException if {@code count} is less than {@code 1}
|
||||||
*/
|
*/
|
||||||
public Builder<V> put(final String key, final V value, final int count) {
|
public Builder<V> put(final String key, final V value, final int count) {
|
||||||
Objects.requireNonNull(key, "key");
|
Objects.requireNonNull(key, ARG_KEY);
|
||||||
Objects.requireNonNull(value, "value");
|
Objects.requireNonNull(value, "value");
|
||||||
|
|
||||||
if (count < 1) { // NOPMD
|
if (count < 1) { // NOPMD
|
||||||
@@ -1098,7 +1460,7 @@ public final class FrequencyTrie<V> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
final CompiledNode<V> frozen = new CompiledNode<>(edges, childNodes, localSummary.orderedValues(),
|
final CompiledNode<V> frozen = new CompiledNode<>(edges, childNodes, localSummary.orderedValues(),
|
||||||
localSummary.orderedCounts());
|
this.maxExpandedIndex, localSummary.orderedCounts());
|
||||||
cache.put(reducedNode, frozen);
|
cache.put(reducedNode, frozen);
|
||||||
return frozen;
|
return frozen;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,11 +119,11 @@ public final class FrequencyTrieBuilders {
|
|||||||
* Copies one compiled node and all reachable descendants into the target
|
* Copies one compiled node and all reachable descendants into the target
|
||||||
* builder.
|
* builder.
|
||||||
*
|
*
|
||||||
* @param node current compiled node
|
* @param node current compiled node
|
||||||
* @param keyBuilder current key builder
|
* @param keyBuilder current key builder
|
||||||
* @param builder target mutable builder
|
* @param builder target mutable builder
|
||||||
* @param traversalDirection logical key traversal direction used by the source
|
* @param traversalDirection logical key traversal direction used by the source
|
||||||
* @param <V> value type
|
* @param <V> value type
|
||||||
*/
|
*/
|
||||||
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
|
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
|
||||||
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {
|
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ import java.util.concurrent.locks.ReentrantLock;
|
|||||||
* instance can still be used safely when needed.
|
* instance can still be used safely when needed.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
@SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", "PMD.CyclomaticComplexity", "PMD.ForLoopVariableCount" })
|
||||||
public final class PatchCommandEncoder {
|
public final class PatchCommandEncoder {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -121,6 +121,13 @@ public final class PatchCommandEncoder {
|
|||||||
*/
|
*/
|
||||||
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
|
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return value used by
|
||||||
|
* {@link #applyTo(CharSequence, String, WordTraversalDirection, char[], int, int)}
|
||||||
|
* when the caller-owned output range is too small for the transformed text.
|
||||||
|
*/
|
||||||
|
public static final int APPLY_INSUFFICIENT_CAPACITY = -1;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prefix used in unsupported NOOP patch argument exceptions.
|
* Prefix used in unsupported NOOP patch argument exceptions.
|
||||||
*/
|
*/
|
||||||
@@ -346,6 +353,78 @@ public final class PatchCommandEncoder {
|
|||||||
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
|
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies a compact patch command into a caller-owned output buffer.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* The output array is not retained. Capacity failure is reported by
|
||||||
|
* {@link #APPLY_INSUFFICIENT_CAPACITY} and leaves the output range unchanged.
|
||||||
|
* Malformed compatibility cases preserve the source exactly as
|
||||||
|
* {@link #apply(String, String, WordTraversalDirection)} does.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param source original source text
|
||||||
|
* @param patchCommand compact patch command
|
||||||
|
* @param traversalDirection traversal direction used by the patch command
|
||||||
|
* @param output caller-owned output storage
|
||||||
|
* @param outputOffset first writable output offset
|
||||||
|
* @param outputLength writable output capacity
|
||||||
|
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
|
||||||
|
* when {@code outputLength} is too small
|
||||||
|
*/
|
||||||
|
public static int applyTo(final CharSequence source, final String patchCommand,
|
||||||
|
final WordTraversalDirection traversalDirection, final char[] output, final int outputOffset,
|
||||||
|
final int outputLength) {
|
||||||
|
Objects.requireNonNull(source, "source");
|
||||||
|
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||||
|
Objects.requireNonNull(output, "output");
|
||||||
|
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
|
||||||
|
|
||||||
|
final int sourceLength = source.length();
|
||||||
|
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
|
||||||
|
if (producedLength > outputLength) {
|
||||||
|
return APPLY_INSUFFICIENT_CAPACITY;
|
||||||
|
}
|
||||||
|
applyToOutput(source, 0, sourceLength, patchCommand, traversalDirection, output, outputOffset, producedLength);
|
||||||
|
return producedLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies a compact patch command from a caller-owned source slice into a
|
||||||
|
* caller-owned output buffer.
|
||||||
|
*
|
||||||
|
* @param source source storage
|
||||||
|
* @param sourceOffset first source character offset
|
||||||
|
* @param sourceLength number of source characters
|
||||||
|
* @param patchCommand compact patch command
|
||||||
|
* @param traversalDirection traversal direction used by the patch command
|
||||||
|
* @param output caller-owned output storage
|
||||||
|
* @param outputOffset first writable output offset
|
||||||
|
* @param outputLength writable output capacity
|
||||||
|
* @return produced character count, or {@link #APPLY_INSUFFICIENT_CAPACITY}
|
||||||
|
* when {@code outputLength} is too small
|
||||||
|
* @throws IllegalArgumentException when source and output ranges overlap in the
|
||||||
|
* same array
|
||||||
|
*/
|
||||||
|
public static int applyTo(final char[] source, final int sourceOffset, final int sourceLength,
|
||||||
|
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||||
|
final int outputOffset, final int outputLength) {
|
||||||
|
Objects.requireNonNull(source, "source");
|
||||||
|
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||||
|
Objects.requireNonNull(output, "output");
|
||||||
|
Objects.checkFromIndexSize(sourceOffset, sourceLength, source.length);
|
||||||
|
Objects.checkFromIndexSize(outputOffset, outputLength, output.length);
|
||||||
|
validateNonOverlappingRanges(source, sourceOffset, sourceLength, output, outputOffset, outputLength);
|
||||||
|
|
||||||
|
final int producedLength = computeAppliedLength(sourceLength, patchCommand, traversalDirection);
|
||||||
|
if (producedLength > outputLength) {
|
||||||
|
return APPLY_INSUFFICIENT_CAPACITY;
|
||||||
|
}
|
||||||
|
applyToOutput(source, sourceOffset, sourceLength, patchCommand, traversalDirection, output, outputOffset,
|
||||||
|
producedLength);
|
||||||
|
return producedLength;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Encodes a patch command using the historical backward Egothor semantics.
|
* Encodes a patch command using the historical backward Egothor semantics.
|
||||||
*
|
*
|
||||||
@@ -409,7 +488,6 @@ public final class PatchCommandEncoder {
|
|||||||
* @param patchCommand compact patch command
|
* @param patchCommand compact patch command
|
||||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
|
||||||
private static String applyBackward(final String source, final String patchCommand) {
|
private static String applyBackward(final String source, final String patchCommand) {
|
||||||
if (source == null) {
|
if (source == null) {
|
||||||
return null;
|
return null;
|
||||||
@@ -435,7 +513,7 @@ public final class PatchCommandEncoder {
|
|||||||
int position = result.length() - 1;
|
int position = result.length() - 1;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
final char opcode = patchCommand.charAt(patchIndex);
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
@@ -493,7 +571,6 @@ public final class PatchCommandEncoder {
|
|||||||
* @param patchCommand compact patch command
|
* @param patchCommand compact patch command
|
||||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
|
||||||
private static String applyForward(final String source, final String patchCommand) {
|
private static String applyForward(final String source, final String patchCommand) {
|
||||||
if (source == null) {
|
if (source == null) {
|
||||||
return null;
|
return null;
|
||||||
@@ -519,7 +596,7 @@ public final class PatchCommandEncoder {
|
|||||||
int position = 0;
|
int position = 0;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
final char opcode = patchCommand.charAt(patchIndex);
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
@@ -681,7 +758,7 @@ public final class PatchCommandEncoder {
|
|||||||
*/
|
*/
|
||||||
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||||
try {
|
try {
|
||||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
final char opcode = patchCommand.charAt(patchIndex);
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
@@ -722,7 +799,7 @@ public final class PatchCommandEncoder {
|
|||||||
*/
|
*/
|
||||||
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||||
try {
|
try {
|
||||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
final char opcode = patchCommand.charAt(patchIndex);
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
@@ -753,6 +830,711 @@ public final class PatchCommandEncoder {
|
|||||||
return result.toString();
|
return result.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes the transformed length or the preserved source length for malformed
|
||||||
|
* compatibility cases.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeAppliedLength(final int sourceLength, final String patchCommand,
|
||||||
|
final WordTraversalDirection traversalDirection) {
|
||||||
|
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|
||||||
|
|| (patchCommand.length() & 1) != 0) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||||
|
return computeBackwardAppliedLength(sourceLength, patchCommand);
|
||||||
|
}
|
||||||
|
return computeForwardAppliedLength(sourceLength, patchCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes the backward traversal output length.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeBackwardAppliedLength(final int sourceLength, final String patchCommand) {
|
||||||
|
if (patchCommand.length() == 2) {
|
||||||
|
return computeSingleBackwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||||
|
}
|
||||||
|
if (sourceLength == 0) {
|
||||||
|
return computeBackwardEmptyAppliedLength(patchCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = sourceLength - 1;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
final int skipCount = decodeEncodedCount(argument);
|
||||||
|
if (skipCount < 1) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
position = position - skipCount + 1;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
if (position < 0 || position >= currentLength) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteCount = decodeEncodedCount(argument);
|
||||||
|
if (deleteCount < 1) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
final int deleteEndExclusive = position + 1;
|
||||||
|
position -= deleteCount - 1;
|
||||||
|
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
currentLength -= deleteEndExclusive - position;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
if (position < -1 || position >= currentLength) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
currentLength++;
|
||||||
|
position++;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return sourceLength;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
|
||||||
|
position--;
|
||||||
|
}
|
||||||
|
return currentLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes the forward traversal output length.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeForwardAppliedLength(final int sourceLength, final String patchCommand) {
|
||||||
|
if (patchCommand.length() == 2) {
|
||||||
|
return computeSingleForwardAppliedLength(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||||
|
}
|
||||||
|
if (sourceLength == 0) {
|
||||||
|
return computeForwardEmptyAppliedLength(patchCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
final int skipCount = decodeEncodedCount(argument);
|
||||||
|
if (skipCount < 1) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
position = position + skipCount - 1;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
if (position < 0 || position >= currentLength) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteCount = decodeEncodedCount(argument);
|
||||||
|
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
currentLength -= deleteCount;
|
||||||
|
position--;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
if (position < 0 || position > currentLength) {
|
||||||
|
return sourceLength;
|
||||||
|
}
|
||||||
|
currentLength++;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return sourceLength;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
|
||||||
|
position++;
|
||||||
|
}
|
||||||
|
return currentLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes a single backward instruction output length.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param opcode opcode
|
||||||
|
* @param argument argument
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeSingleBackwardAppliedLength(final int sourceLength, final char opcode,
|
||||||
|
final char argument) {
|
||||||
|
final int encodedValue;
|
||||||
|
switch (opcode) {
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
encodedValue = decodeEncodedCount(argument);
|
||||||
|
return encodedValue < 1 || encodedValue > sourceLength ? sourceLength : sourceLength - encodedValue;
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
return sourceLength + 1;
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
return sourceLength;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return sourceLength;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes a single forward instruction output length.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param opcode opcode
|
||||||
|
* @param argument argument
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeSingleForwardAppliedLength(final int sourceLength, final char opcode,
|
||||||
|
final char argument) {
|
||||||
|
return computeSingleBackwardAppliedLength(sourceLength, opcode, argument);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes output length for an empty source in backward traversal.
|
||||||
|
*
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeBackwardEmptyAppliedLength(final String patchCommand) {
|
||||||
|
int currentLength = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
switch (opcode) {
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
currentLength++;
|
||||||
|
break;
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
return 0;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return currentLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes output length for an empty source in forward traversal.
|
||||||
|
*
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return produced length
|
||||||
|
*/
|
||||||
|
private static int computeForwardEmptyAppliedLength(final String patchCommand) {
|
||||||
|
return computeBackwardEmptyAppliedLength(patchCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies an already-sized patch into caller output.
|
||||||
|
*
|
||||||
|
* @param source source text
|
||||||
|
* @param sourceOffset source offset
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
* @param producedLength already-validated produced length
|
||||||
|
*/
|
||||||
|
private static void applyToOutput(final CharSequence source, final int sourceOffset, final int sourceLength,
|
||||||
|
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||||
|
final int outputOffset, final int producedLength) {
|
||||||
|
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
|
||||||
|
copySource(source, sourceOffset, sourceLength, output, outputOffset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sourceLength > 0) {
|
||||||
|
copySource(source, sourceOffset, sourceLength, output, outputOffset);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||||
|
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||||
|
} else {
|
||||||
|
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies an already-sized patch into caller output.
|
||||||
|
*
|
||||||
|
* @param source source storage
|
||||||
|
* @param sourceOffset source offset
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
* @param producedLength already-validated produced length
|
||||||
|
*/
|
||||||
|
private static void applyToOutput(final char[] source, final int sourceOffset, final int sourceLength,
|
||||||
|
final String patchCommand, final WordTraversalDirection traversalDirection, final char[] output,
|
||||||
|
final int outputOffset, final int producedLength) {
|
||||||
|
if (isPreservedSource(sourceLength, producedLength, patchCommand, traversalDirection)) {
|
||||||
|
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sourceLength > 0) {
|
||||||
|
System.arraycopy(source, sourceOffset, output, outputOffset, sourceLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||||
|
applyBackwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||||
|
} else {
|
||||||
|
applyForwardToOutput(sourceLength, patchCommand, output, outputOffset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines whether the output is exactly the original source.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param producedLength produced length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @return {@code true} if copying the source is sufficient
|
||||||
|
*/
|
||||||
|
private static boolean isPreservedSource(final int sourceLength, final int producedLength,
|
||||||
|
final String patchCommand, final WordTraversalDirection traversalDirection) {
|
||||||
|
return producedLength == sourceLength
|
||||||
|
&& isKnownPreserveOnlyPatch(sourceLength, patchCommand, traversalDirection);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns whether equal length also means no mutation is needed.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @return {@code true} when the command preserves source content
|
||||||
|
*/
|
||||||
|
private static boolean isKnownPreserveOnlyPatch(final int sourceLength, final String patchCommand,
|
||||||
|
final WordTraversalDirection traversalDirection) {
|
||||||
|
if (patchCommand == null || patchCommand.isEmpty() || NOOP_PATCH.equals(patchCommand)
|
||||||
|
|| (patchCommand.length() & 1) != 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (patchCommand.length() == 2) {
|
||||||
|
return isSingleInstructionPreserveOnly(sourceLength, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||||
|
}
|
||||||
|
if (sourceLength == 0) {
|
||||||
|
return hasEmptySourcePreserveOnlyPatch(patchCommand);
|
||||||
|
}
|
||||||
|
return traversalDirection == WordTraversalDirection.BACKWARD
|
||||||
|
? hasBackwardPreserveOnlyPatch(sourceLength, patchCommand)
|
||||||
|
: hasForwardPreserveOnlyPatch(sourceLength, patchCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests whether a single instruction preserves the source content.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param opcode opcode
|
||||||
|
* @param argument argument
|
||||||
|
* @return {@code true} when no mutation should be applied
|
||||||
|
*/
|
||||||
|
private static boolean isSingleInstructionPreserveOnly(final int sourceLength, final char opcode,
|
||||||
|
final char argument) {
|
||||||
|
switch (opcode) {
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int encodedValue = decodeEncodedCount(argument);
|
||||||
|
return encodedValue < 1 || encodedValue > sourceLength;
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
return false;
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
return sourceLength == 0;
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
return true;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests whether an empty-source patch preserves the source.
|
||||||
|
*
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return {@code true} when no mutation should be applied
|
||||||
|
*/
|
||||||
|
private static boolean hasEmptySourcePreserveOnlyPatch(final String patchCommand) {
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
switch (opcode) {
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
break;
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
return true;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests whether a backward patch preserves the source because it is malformed
|
||||||
|
* or a NOOP.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return {@code true} when no mutation should be applied
|
||||||
|
*/
|
||||||
|
private static boolean hasBackwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = sourceLength - 1;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
final int skipCount = decodeEncodedCount(argument);
|
||||||
|
if (skipCount < 1) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
position = position - skipCount + 1;
|
||||||
|
break;
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
if (position < 0 || position >= currentLength) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteCount = decodeEncodedCount(argument);
|
||||||
|
if (deleteCount < 1) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
final int deleteEndExclusive = position + 1;
|
||||||
|
position -= deleteCount - 1;
|
||||||
|
if (position < 0 || deleteEndExclusive > currentLength || position > deleteEndExclusive) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
currentLength -= deleteEndExclusive - position;
|
||||||
|
break;
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
if (position < -1 || position >= currentLength) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
currentLength++;
|
||||||
|
position++;
|
||||||
|
break;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
position--;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests whether a forward patch preserves the source because it is malformed or
|
||||||
|
* a NOOP.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @return {@code true} when no mutation should be applied
|
||||||
|
*/
|
||||||
|
private static boolean hasForwardPreserveOnlyPatch(final int sourceLength, final String patchCommand) {
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
final int skipCount = decodeEncodedCount(argument);
|
||||||
|
if (skipCount < 1) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
position = position + skipCount - 1;
|
||||||
|
break;
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
if (position < 0 || position >= currentLength) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteCount = decodeEncodedCount(argument);
|
||||||
|
if (deleteCount < 1 || position < 0 || position + deleteCount > currentLength) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
currentLength -= deleteCount;
|
||||||
|
position--;
|
||||||
|
break;
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
if (position < 0 || position > currentLength) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
currentLength++;
|
||||||
|
break;
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
if (argument != NOOP_ARGUMENT) {
|
||||||
|
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||||
|
}
|
||||||
|
position++;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Copies source characters from a sequence.
|
||||||
|
*
|
||||||
|
* @param source source text
|
||||||
|
* @param sourceOffset source offset
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
*/
|
||||||
|
private static void copySource(final CharSequence source, final int sourceOffset, final int sourceLength,
|
||||||
|
final char[] output, final int outputOffset) {
|
||||||
|
for (int index = 0; index < sourceLength; index++) {
|
||||||
|
output[outputOffset + index] = source.charAt(sourceOffset + index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies a backward patch after validation.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param output output storage initialized with source
|
||||||
|
* @param outputOffset output offset
|
||||||
|
*/
|
||||||
|
private static void applyBackwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
|
||||||
|
final int outputOffset) {
|
||||||
|
if (sourceLength == 0) {
|
||||||
|
applyBackwardEmptyToOutput(patchCommand, output, outputOffset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = sourceLength - 1;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
position = position - decodeEncodedCount(argument) + 1;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
output[outputOffset + position] = argument;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteEndExclusive = position + 1;
|
||||||
|
position -= decodeEncodedCount(argument) - 1;
|
||||||
|
System.arraycopy(output, outputOffset + deleteEndExclusive, output, outputOffset + position,
|
||||||
|
currentLength - deleteEndExclusive);
|
||||||
|
currentLength -= deleteEndExclusive - position;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
final int insertIndex = position + 1;
|
||||||
|
System.arraycopy(output, outputOffset + insertIndex, output, outputOffset + insertIndex + 1,
|
||||||
|
currentLength - insertIndex);
|
||||||
|
output[outputOffset + insertIndex] = argument;
|
||||||
|
currentLength++;
|
||||||
|
position++;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
return;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new AssertionError("Patch command was not validated.");
|
||||||
|
}
|
||||||
|
|
||||||
|
position--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies a forward patch after validation.
|
||||||
|
*
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param output output storage initialized with source
|
||||||
|
* @param outputOffset output offset
|
||||||
|
*/
|
||||||
|
private static void applyForwardToOutput(final int sourceLength, final String patchCommand, final char[] output,
|
||||||
|
final int outputOffset) {
|
||||||
|
if (sourceLength == 0) {
|
||||||
|
applyForwardEmptyToOutput(patchCommand, output, outputOffset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int currentLength = sourceLength;
|
||||||
|
int position = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char opcode = patchCommand.charAt(patchIndex);
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
|
||||||
|
switch (opcode) {
|
||||||
|
case SKIP_OPCODE:
|
||||||
|
position = position + decodeEncodedCount(argument) - 1;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case REPLACE_OPCODE:
|
||||||
|
output[outputOffset + position] = argument;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case DELETE_OPCODE:
|
||||||
|
final int deleteCount = decodeEncodedCount(argument);
|
||||||
|
System.arraycopy(output, outputOffset + position + deleteCount, output, outputOffset + position,
|
||||||
|
currentLength - position - deleteCount);
|
||||||
|
currentLength -= deleteCount;
|
||||||
|
position--;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case INSERT_OPCODE:
|
||||||
|
System.arraycopy(output, outputOffset + position, output, outputOffset + position + 1,
|
||||||
|
currentLength - position);
|
||||||
|
output[outputOffset + position] = argument;
|
||||||
|
currentLength++;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case NOOP_OPCODE:
|
||||||
|
return;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new AssertionError("Patch command was not validated.");
|
||||||
|
}
|
||||||
|
|
||||||
|
position++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies an empty-source backward patch after validation.
|
||||||
|
*
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
*/
|
||||||
|
private static void applyBackwardEmptyToOutput(final String patchCommand, final char[] output,
|
||||||
|
final int outputOffset) {
|
||||||
|
int currentLength = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||||
|
System.arraycopy(output, outputOffset, output, outputOffset + 1, currentLength);
|
||||||
|
output[outputOffset] = argument;
|
||||||
|
currentLength++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies an empty-source forward patch after validation.
|
||||||
|
*
|
||||||
|
* @param patchCommand patch command
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
*/
|
||||||
|
private static void applyForwardEmptyToOutput(final String patchCommand, final char[] output,
|
||||||
|
final int outputOffset) {
|
||||||
|
int currentLength = 0;
|
||||||
|
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) {
|
||||||
|
output[outputOffset + currentLength] = patchCommand.charAt(patchIndex + 1);
|
||||||
|
currentLength++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates that source and output slices do not overlap when backed by the
|
||||||
|
* same array.
|
||||||
|
*
|
||||||
|
* @param source source storage
|
||||||
|
* @param sourceOffset source offset
|
||||||
|
* @param sourceLength source length
|
||||||
|
* @param output output storage
|
||||||
|
* @param outputOffset output offset
|
||||||
|
* @param outputLength output length
|
||||||
|
*/
|
||||||
|
private static void validateNonOverlappingRanges(final char[] source, final int sourceOffset,
|
||||||
|
final int sourceLength, final char[] output, final int outputOffset, final int outputLength) {
|
||||||
|
if (!source.equals(output) || sourceLength == 0 || outputLength == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
final int sourceEnd = sourceOffset + sourceLength;
|
||||||
|
final int outputEnd = outputOffset + outputLength;
|
||||||
|
if (sourceOffset < outputEnd && outputOffset < sourceEnd) {
|
||||||
|
throw new IllegalArgumentException("source and output ranges must not overlap.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the direction-specialized apply strategy.
|
* Returns the direction-specialized apply strategy.
|
||||||
*
|
*
|
||||||
@@ -769,7 +1551,6 @@ public final class PatchCommandEncoder {
|
|||||||
* @param argument serialized count argument
|
* @param argument serialized count argument
|
||||||
* @return decoded positive count, or {@code -1} when the argument is malformed
|
* @return decoded positive count, or {@code -1} when the argument is malformed
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
|
||||||
private static int decodeEncodedCount(final char argument) {
|
private static int decodeEncodedCount(final char argument) {
|
||||||
if (argument < 'a') {
|
if (argument < 'a') {
|
||||||
return -1;
|
return -1;
|
||||||
|
|||||||
@@ -94,6 +94,29 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a GZip-compressed binary patch-command trie from a filesystem path with
|
||||||
|
* an optional dense child lookup span override.
|
||||||
|
* <p>
|
||||||
|
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||||
|
* persisted in the file and does not change the compiled metadata.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param path source file
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return deserialized trie
|
||||||
|
* @throws NullPointerException if {@code path} is {@code null}
|
||||||
|
* @throws IOException if reading or decompression fails
|
||||||
|
*/
|
||||||
|
public static FrequencyTrie<String> read(final Path path, final int maxExpandedIndex) throws IOException {
|
||||||
|
Objects.requireNonNull(path, "path");
|
||||||
|
|
||||||
|
try (InputStream fileInputStream = Files.newInputStream(path)) {
|
||||||
|
return read(fileInputStream, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads a GZip-compressed binary patch-command trie from a filesystem path
|
* Reads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
* string.
|
* string.
|
||||||
@@ -108,6 +131,26 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
return read(Path.of(fileName));
|
return read(Path.of(fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
|
* string with an optional dense child lookup span override.
|
||||||
|
* <p>
|
||||||
|
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||||
|
* persisted in the file and does not change the compiled metadata.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param fileName source file name or path string
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return deserialized trie
|
||||||
|
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||||
|
* @throws IOException if reading or decompression fails
|
||||||
|
*/
|
||||||
|
public static FrequencyTrie<String> read(final String fileName, final int maxExpandedIndex) throws IOException {
|
||||||
|
Objects.requireNonNull(fileName, "fileName");
|
||||||
|
return read(Path.of(fileName), maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads a GZip-compressed binary patch-command trie from an input stream.
|
* Reads a GZip-compressed binary patch-command trie from an input stream.
|
||||||
*
|
*
|
||||||
@@ -132,6 +175,35 @@ public final class StemmerPatchTrieBinaryIO {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a GZip-compressed binary patch-command trie from an input stream with
|
||||||
|
* an optional dense child lookup span override.
|
||||||
|
* <p>
|
||||||
|
* This is a runtime-only tuning parameter. The dense-span setting is not
|
||||||
|
* persisted in the file and does not change the compiled metadata.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param inputStream source stream
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return deserialized trie
|
||||||
|
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||||
|
* @throws IOException if reading or decompression fails
|
||||||
|
*/
|
||||||
|
public static FrequencyTrie<String> read(final InputStream inputStream, final int maxExpandedIndex)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(inputStream, "inputStream");
|
||||||
|
|
||||||
|
try (GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(inputStream));
|
||||||
|
DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) {
|
||||||
|
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(dataInputStream, String[]::new, STRING_CODEC,
|
||||||
|
maxExpandedIndex);
|
||||||
|
|
||||||
|
LOGGER.log(Level.FINE, "Read compressed binary stemmer trie.");
|
||||||
|
return trie;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||||
* at a filesystem path.
|
* at a filesystem path.
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ import java.util.zip.GZIPInputStream;
|
|||||||
public final class StemmerPatchTrieLoader {
|
public final class StemmerPatchTrieLoader {
|
||||||
|
|
||||||
/* default */ static final String FILENAME_REQUIRED = "fileName required";
|
/* default */ static final String FILENAME_REQUIRED = "fileName required";
|
||||||
|
private static final String PARAMETER_PATH = "path";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Logger of this class.
|
* Logger of this class.
|
||||||
@@ -461,7 +462,7 @@ public final class StemmerPatchTrieLoader {
|
|||||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
|
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
|
||||||
diacriticProcessingMode);
|
diacriticProcessingMode);
|
||||||
return load(path, storeOriginal, metadata);
|
return load(path, storeOriginal, metadata);
|
||||||
@@ -487,7 +488,7 @@ public final class StemmerPatchTrieLoader {
|
|||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
|
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
Objects.requireNonNull(metadata, "metadata");
|
Objects.requireNonNull(metadata, "metadata");
|
||||||
|
|
||||||
try (InputStream inputStream = openDictionaryInputStream(path);
|
try (InputStream inputStream = openDictionaryInputStream(path);
|
||||||
@@ -759,10 +760,31 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* read
|
* read
|
||||||
*/
|
*/
|
||||||
public static FrequencyTrie<String> loadBinary(final Path path) throws IOException {
|
public static FrequencyTrie<String> loadBinary(final Path path) throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
return StemmerPatchTrieBinaryIO.read(path);
|
return StemmerPatchTrieBinaryIO.read(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
|
* using a custom dense lookup span override.
|
||||||
|
* <p>
|
||||||
|
* This is a runtime-only tuning parameter that does not affect persisted
|
||||||
|
* metadata.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param path path to the compressed binary trie file
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code path} is {@code null}
|
||||||
|
* @throws IOException if the file cannot be opened, decompressed, or
|
||||||
|
* read
|
||||||
|
*/
|
||||||
|
public static FrequencyTrie<String> loadBinary(final Path path, final int maxExpandedIndex) throws IOException {
|
||||||
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
|
return StemmerPatchTrieBinaryIO.read(path, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
* string.
|
* string.
|
||||||
@@ -778,6 +800,28 @@ public final class StemmerPatchTrieLoader {
|
|||||||
return StemmerPatchTrieBinaryIO.read(fileName);
|
return StemmerPatchTrieBinaryIO.read(fileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads a GZip-compressed binary patch-command trie from a filesystem path
|
||||||
|
* string using a custom dense lookup span override.
|
||||||
|
* <p>
|
||||||
|
* This is a runtime-only tuning parameter that does not affect persisted
|
||||||
|
* metadata.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* @param fileName file name or path string
|
||||||
|
* @param maxExpandedIndex dense lookup span override; negative values use
|
||||||
|
* {@link FrequencyTrie#DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* @return compiled patch-command trie
|
||||||
|
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||||
|
* @throws IOException if the file cannot be opened, decompressed, or
|
||||||
|
* read
|
||||||
|
*/
|
||||||
|
public static FrequencyTrie<String> loadBinary(final String fileName, final int maxExpandedIndex)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||||
|
return StemmerPatchTrieBinaryIO.read(fileName, maxExpandedIndex);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads a GZip-compressed binary patch-command trie from an input stream.
|
* Loads a GZip-compressed binary patch-command trie from an input stream.
|
||||||
*
|
*
|
||||||
@@ -802,7 +846,7 @@ public final class StemmerPatchTrieLoader {
|
|||||||
* read
|
* read
|
||||||
*/
|
*/
|
||||||
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
|
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
return StemmerPatchTrieBinaryIO.readMetadata(path);
|
return StemmerPatchTrieBinaryIO.readMetadata(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -845,7 +889,7 @@ public final class StemmerPatchTrieLoader {
|
|||||||
*/
|
*/
|
||||||
public static void saveBinary(final FrequencyTrie<String> trie, final Path path) throws IOException {
|
public static void saveBinary(final FrequencyTrie<String> trie, final Path path) throws IOException {
|
||||||
Objects.requireNonNull(trie, "trie");
|
Objects.requireNonNull(trie, "trie");
|
||||||
Objects.requireNonNull(path, "path");
|
Objects.requireNonNull(path, PARAMETER_PATH);
|
||||||
StemmerPatchTrieBinaryIO.write(trie, path);
|
StemmerPatchTrieBinaryIO.write(trie, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -58,17 +58,17 @@
|
|||||||
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
|
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
|
||||||
* traditional line-oriented tab-separated values resource format in which each
|
* traditional line-oriented tab-separated values resource format in which each
|
||||||
* non-empty logical line starts with a canonical stem followed by known surface
|
* non-empty logical line starts with a canonical stem followed by known surface
|
||||||
* variants in subsequent tab-separated columns.
|
* variants in subsequent tab-separated columns. Parsing is delegated to
|
||||||
* Parsing is delegated to {@link org.egothor.stemmer.StemmerDictionaryParser},
|
* {@link org.egothor.stemmer.StemmerDictionaryParser}, which applies
|
||||||
* which applies configurable case processing through
|
* configurable case processing through
|
||||||
* {@link org.egothor.stemmer.CaseProcessingMode} (default:
|
* {@link org.egothor.stemmer.CaseProcessingMode} (default:
|
||||||
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
|
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
|
||||||
* supports whole-line as well as trailing remarks introduced by {@code #} or
|
* supports whole-line as well as trailing remarks introduced by {@code #} or
|
||||||
* {@code //}, and currently ignores dictionary items containing Unicode
|
* {@code //}, and currently ignores dictionary items containing Unicode
|
||||||
* whitespace characters while reporting them through warning-level diagnostics.
|
* whitespace characters while reporting them through warning-level diagnostics.
|
||||||
* During loading, each variant is converted into a patch command
|
* During loading, each variant is converted into a patch command targeting the
|
||||||
* targeting the canonical stem, and the stem itself may optionally be stored
|
* canonical stem, and the stem itself may optionally be stored under the
|
||||||
* under the canonical no-operation patch.
|
* canonical no-operation patch.
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* <p>
|
* <p>
|
||||||
|
|||||||
@@ -43,14 +43,15 @@ import java.util.Objects;
|
|||||||
* immutable from the public API perspective because construction wires these
|
* immutable from the public API perspective because construction wires these
|
||||||
* arrays once and all lookup operations thereafter treat them as read-only.
|
* arrays once and all lookup operations thereafter treat them as read-only.
|
||||||
*
|
*
|
||||||
* @param <V> value type
|
* @param <V> value type
|
||||||
* @param edgeLabels internal edge label array
|
|
||||||
* @param children internal child array
|
|
||||||
* @param orderedValues internal ordered values array
|
|
||||||
* @param orderedCounts internal ordered counts array
|
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.DataClass")
|
public final class CompiledNode<V> {
|
||||||
public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[] orderedValues, int... orderedCounts) {
|
|
||||||
|
/**
|
||||||
|
* Default dense child lookup span in characters used when an explicit override
|
||||||
|
* is not provided.
|
||||||
|
*/
|
||||||
|
public static final int DEFAULT_MAX_EXPANDED_INDEX = 512;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Number of child edges where linear scan is cheaper than binary search.
|
* Number of child edges where linear scan is cheaper than binary search.
|
||||||
@@ -58,24 +59,112 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4;
|
private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates one validated compiled node.
|
* Edge labels in sorted ascending order.
|
||||||
|
*/
|
||||||
|
private final char[] edgeLabels;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sparse child array aligned with {@link #edgeLabels}.
|
||||||
|
*/
|
||||||
|
private final CompiledNode<V>[] children;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dense child lookup table used when labels fit into a compact char interval.
|
||||||
|
* <p>
|
||||||
|
* The table enables direct O(1) indexing for child lookup and is allocated only
|
||||||
|
* when the character span of this node's edges is within the configured
|
||||||
|
* threshold.
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
private final CompiledNode<V>[] denseChildren;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalized minimum edge value for the dense lookup table.
|
||||||
|
*/
|
||||||
|
private final int denseEdgeMin;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Values stored at this node in local order.
|
||||||
|
*/
|
||||||
|
private final V[] orderedValues;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Occurrence counts aligned with {@link #orderedValues}.
|
||||||
|
*/
|
||||||
|
private final int[] orderedCounts;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates one validated compiled node using {@link #DEFAULT_MAX_EXPANDED_INDEX}
|
||||||
|
* for dense lookup sizing.
|
||||||
*
|
*
|
||||||
* @throws NullPointerException if any array argument is {@code null}
|
* @throws NullPointerException if any array argument is {@code null}
|
||||||
* @throws IllegalArgumentException if the edge-related arrays or value-related
|
* @throws IllegalArgumentException if the edge-related arrays or value-related
|
||||||
* arrays do not have matching lengths
|
* arrays do not have matching lengths
|
||||||
*/
|
*/
|
||||||
public CompiledNode {
|
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
|
||||||
|
final int... orderedCounts) {
|
||||||
|
this(edgeLabels, children, orderedValues, DEFAULT_MAX_EXPANDED_INDEX, orderedCounts);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates one validated compiled node.
|
||||||
|
*
|
||||||
|
* @param maxExpandedIndex upper bound for the dense lookup interval size; zero
|
||||||
|
* disables dense lookup. Larger values improve
|
||||||
|
* direct-index likelihood while increasing dense table
|
||||||
|
* memory in compact-label nodes.
|
||||||
|
* @throws NullPointerException if any array argument is {@code null}
|
||||||
|
* @throws IllegalArgumentException if the edge-related arrays or value-related
|
||||||
|
* arrays do not have matching lengths or the
|
||||||
|
* dense interval size is negative
|
||||||
|
*/
|
||||||
|
public CompiledNode(final char[] edgeLabels, final CompiledNode<V>[] children, final V[] orderedValues,
|
||||||
|
final int maxExpandedIndex, final int... orderedCounts) {
|
||||||
Objects.requireNonNull(edgeLabels, "edgeLabels");
|
Objects.requireNonNull(edgeLabels, "edgeLabels");
|
||||||
Objects.requireNonNull(children, "children");
|
Objects.requireNonNull(children, "children");
|
||||||
Objects.requireNonNull(orderedValues, "orderedValues");
|
Objects.requireNonNull(orderedValues, "orderedValues");
|
||||||
Objects.requireNonNull(orderedCounts, "orderedCounts");
|
Objects.requireNonNull(orderedCounts, "orderedCounts");
|
||||||
|
|
||||||
|
if (maxExpandedIndex < 0) {
|
||||||
|
throw new IllegalArgumentException("maxExpandedIndex must be non-negative.");
|
||||||
|
}
|
||||||
|
|
||||||
if (edgeLabels.length != children.length) {
|
if (edgeLabels.length != children.length) {
|
||||||
throw new IllegalArgumentException("edgeLabels and children must have the same length.");
|
throw new IllegalArgumentException("edgeLabels and children must have the same length.");
|
||||||
}
|
}
|
||||||
if (orderedValues.length != orderedCounts.length) {
|
if (orderedValues.length != orderedCounts.length) {
|
||||||
throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length.");
|
throw new IllegalArgumentException("orderedValues and orderedCounts must have the same length.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.edgeLabels = edgeLabels;
|
||||||
|
this.children = children;
|
||||||
|
this.orderedValues = orderedValues;
|
||||||
|
this.orderedCounts = orderedCounts;
|
||||||
|
|
||||||
|
if (edgeLabels.length == 0 || maxExpandedIndex == 0) {
|
||||||
|
this.denseChildren = null;
|
||||||
|
this.denseEdgeMin = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int minEdge = edgeLabels[0];
|
||||||
|
final int maxEdge = edgeLabels[edgeLabels.length - 1];
|
||||||
|
final int span = maxEdge - minEdge;
|
||||||
|
|
||||||
|
if (span < 0 || span > maxExpandedIndex) {
|
||||||
|
this.denseChildren = null;
|
||||||
|
this.denseEdgeMin = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
final CompiledNode<V>[] dense = (CompiledNode<V>[]) new CompiledNode[span + 1];
|
||||||
|
for (int edgeIndex = 0; edgeIndex < edgeLabels.length; edgeIndex++) {
|
||||||
|
dense[edgeLabels[edgeIndex] - minEdge] = children[edgeIndex];
|
||||||
|
}
|
||||||
|
|
||||||
|
this.denseChildren = dense;
|
||||||
|
this.denseEdgeMin = minEdge;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -87,7 +176,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
*
|
*
|
||||||
* @return internal edge-label array
|
* @return internal edge-label array
|
||||||
*/
|
*/
|
||||||
@Override
|
|
||||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||||
public char[] edgeLabels() {
|
public char[] edgeLabels() {
|
||||||
return this.edgeLabels;
|
return this.edgeLabels;
|
||||||
@@ -102,7 +190,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
*
|
*
|
||||||
* @return internal child-node array
|
* @return internal child-node array
|
||||||
*/
|
*/
|
||||||
@Override
|
|
||||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||||
public CompiledNode<V>[] children() {
|
public CompiledNode<V>[] children() {
|
||||||
return this.children;
|
return this.children;
|
||||||
@@ -117,7 +204,6 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
*
|
*
|
||||||
* @return internal ordered-values array
|
* @return internal ordered-values array
|
||||||
*/
|
*/
|
||||||
@Override
|
|
||||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||||
public V[] orderedValues() {
|
public V[] orderedValues() {
|
||||||
return this.orderedValues;
|
return this.orderedValues;
|
||||||
@@ -132,14 +218,144 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
*
|
*
|
||||||
* @return internal ordered-counts array
|
* @return internal ordered-counts array
|
||||||
*/
|
*/
|
||||||
@Override
|
|
||||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||||
public int[] orderedCounts() {
|
public int[] orderedCounts() {
|
||||||
return this.orderedCounts;
|
return this.orderedCounts;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the number of child edges represented by this node.
|
||||||
|
*
|
||||||
|
* @return child edge count
|
||||||
|
*/
|
||||||
|
public int edgeCount() {
|
||||||
|
return this.edgeLabels.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the number of values stored in this node.
|
||||||
|
*
|
||||||
|
* @return value count
|
||||||
|
*/
|
||||||
|
public int valueCount() {
|
||||||
|
return this.orderedValues.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicates whether this node stores any values.
|
||||||
|
*
|
||||||
|
* @return {@code true} when values are present at this node
|
||||||
|
*/
|
||||||
|
public boolean hasValues() {
|
||||||
|
return this.orderedValues.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicates whether this node has child edges.
|
||||||
|
*
|
||||||
|
* @return {@code true} when this node has at least one outgoing edge
|
||||||
|
*/
|
||||||
|
public boolean hasChildren() {
|
||||||
|
return this.edgeLabels.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicates whether this node has no child edges.
|
||||||
|
*
|
||||||
|
* @return {@code true} when this node is a terminal leaf node
|
||||||
|
*/
|
||||||
|
public boolean isLeaf() {
|
||||||
|
return !hasChildren();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests whether an edge label is present at this node.
|
||||||
|
*
|
||||||
|
* @param edge edge label
|
||||||
|
* @return {@code true} if this node contains the supplied edge label
|
||||||
|
*/
|
||||||
|
public boolean hasEdge(final char edge) {
|
||||||
|
return findChild(edge) != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicates whether this node has a dense direct-index child lookup table.
|
||||||
|
*
|
||||||
|
* @return {@code true} when a direct-index child table is available
|
||||||
|
*/
|
||||||
|
public boolean hasDenseLookup() {
|
||||||
|
return this.denseChildren != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a small memory-related metric describing this node's dense table
|
||||||
|
* size.
|
||||||
|
*
|
||||||
|
* @return number of dense table slots, or {@code 0} when dense lookup is not
|
||||||
|
* enabled
|
||||||
|
*/
|
||||||
|
public int denseTableLength() {
|
||||||
|
return this.denseChildren == null ? 0 : this.denseChildren.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a compact structural summary used by diagnostics and tests.
|
||||||
|
*
|
||||||
|
* @return summary hash for node structure and contents
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
int hash = Arrays.hashCode(this.edgeLabels);
|
||||||
|
hash = 31 * hash + Arrays.hashCode(this.children);
|
||||||
|
hash = 31 * hash + Arrays.hashCode(this.orderedValues);
|
||||||
|
hash = 31 * hash + Arrays.hashCode(this.orderedCounts);
|
||||||
|
hash = 31 * hash + Objects.hash(this.denseEdgeMin);
|
||||||
|
hash = 31 * hash + (hasDenseLookup() ? Arrays.hashCode(this.denseChildren) : 0);
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compares structural node content, including dense table availability.
|
||||||
|
*
|
||||||
|
* @param object comparison object
|
||||||
|
* @return {@code true} when nodes describe identical structure and payload
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public boolean equals(final Object object) {
|
||||||
|
if (this == object) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!(object instanceof CompiledNode<?> other)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return Arrays.equals(this.edgeLabels, other.edgeLabels) && Arrays.equals(this.children, other.children)
|
||||||
|
&& Arrays.equals(this.orderedValues, other.orderedValues)
|
||||||
|
&& Arrays.equals(this.orderedCounts, other.orderedCounts) && this.denseEdgeMin == other.denseEdgeMin
|
||||||
|
&& Arrays.equals(this.denseChildren, other.denseChildren);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a short summary useful for debugging and diagnostics.
|
||||||
|
*
|
||||||
|
* @return textual node summary
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "CompiledNode{" + "edgeCount=" + this.edgeLabels.length + ", orderedValueCount="
|
||||||
|
+ this.orderedValues.length + ", denseTableLength=" + denseTableLength() + '}';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finds a child for the supplied edge character.
|
* Finds a child for the supplied edge character.
|
||||||
|
* <p>
|
||||||
|
* Lookup order is:
|
||||||
|
* <ol>
|
||||||
|
* <li>dense array index (if the label interval is compact enough),</li>
|
||||||
|
* <li>small-child linear scan when the fallback node has
|
||||||
|
* {@value #LINEAR_CHILD_COUNT_THRESHOLD} or fewer edges,</li>
|
||||||
|
* <li>binary search over sorted labels.</li>
|
||||||
|
* </ol>
|
||||||
|
* </p>
|
||||||
*
|
*
|
||||||
* @param edge edge character
|
* @param edge edge character
|
||||||
* @return child node, or {@code null} if absent
|
* @return child node, or {@code null} if absent
|
||||||
@@ -149,6 +365,15 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
|||||||
if (childCount == 0) {
|
if (childCount == 0) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.denseChildren != null) {
|
||||||
|
final int denseIndex = edge - this.denseEdgeMin;
|
||||||
|
if (denseIndex < 0 || denseIndex >= this.denseChildren.length) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return this.denseChildren[denseIndex];
|
||||||
|
}
|
||||||
|
|
||||||
if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) {
|
if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) {
|
||||||
for (int index = 0; index < childCount; index++) {
|
for (int index = 0; index < childCount; index++) {
|
||||||
if (this.edgeLabels[index] == edge) {
|
if (this.edgeLabels[index] == edge) {
|
||||||
|
|||||||
@@ -95,6 +95,9 @@ import org.junit.jupiter.params.provider.MethodSource;
|
|||||||
@Tag("integration")
|
@Tag("integration")
|
||||||
@Tag("cli")
|
@Tag("cli")
|
||||||
@Tag("stemmer")
|
@Tag("stemmer")
|
||||||
|
@Tag("compile")
|
||||||
|
@Tag("construction")
|
||||||
|
@Tag("slow")
|
||||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||||
@DisplayName("Compile integration")
|
@DisplayName("Compile integration")
|
||||||
final class CompileIntegrationTest {
|
final class CompileIntegrationTest {
|
||||||
@@ -182,6 +185,11 @@ final class CompileIntegrationTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("Remark-aware fixture workflow")
|
@DisplayName("Remark-aware fixture workflow")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("cli")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("compile")
|
||||||
|
@Tag("construction")
|
||||||
final class RemarkAwareFixtureWorkflow {
|
final class RemarkAwareFixtureWorkflow {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -189,9 +197,10 @@ final class CompileIntegrationTest {
|
|||||||
* create nested output directories, preserve expected lookup behavior, and
|
* create nested output directories, preserve expected lookup behavior, and
|
||||||
* store canonical stems when {@code --store-original} is enabled.
|
* store canonical stems when {@code --store-original} is enabled.
|
||||||
*
|
*
|
||||||
* @throws IOException if reading or writing fails
|
* @throws IOException if reading or writing fails
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
|
@Tag("slow")
|
||||||
@DisplayName("CLI should compile the remark-aware fixture and preserve expected lookups")
|
@DisplayName("CLI should compile the remark-aware fixture and preserve expected lookups")
|
||||||
void shouldCompileRemarkAwareFixtureAndPreserveExpectedLookups() throws IOException {
|
void shouldCompileRemarkAwareFixtureAndPreserveExpectedLookups() throws IOException {
|
||||||
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
||||||
@@ -234,9 +243,10 @@ final class CompileIntegrationTest {
|
|||||||
* Verifies that the CLI rejects an already existing output path unless
|
* Verifies that the CLI rejects an already existing output path unless
|
||||||
* overwrite is explicitly enabled.
|
* overwrite is explicitly enabled.
|
||||||
*
|
*
|
||||||
* @throws IOException if reading or writing fails
|
* @throws IOException if reading or writing fails
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
|
@Tag("slow")
|
||||||
@DisplayName("CLI should require overwrite before replacing an existing output artifact")
|
@DisplayName("CLI should require overwrite before replacing an existing output artifact")
|
||||||
void shouldRequireOverwriteForExistingOutput() throws IOException {
|
void shouldRequireOverwriteForExistingOutput() throws IOException {
|
||||||
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
final Path inputFile = copyResourceToTemporaryFile(REMARK_AWARE_DICTIONARY_RESOURCE,
|
||||||
@@ -301,6 +311,12 @@ final class CompileIntegrationTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("Bundled project dictionary workflows")
|
@DisplayName("Bundled project dictionary workflows")
|
||||||
|
@Tag("slow")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("cli")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("compile")
|
||||||
|
@Tag("construction")
|
||||||
final class BundledProjectDictionaryWorkflows {
|
final class BundledProjectDictionaryWorkflows {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -317,11 +333,12 @@ final class CompileIntegrationTest {
|
|||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* @param scenario scenario identifier
|
* @param scenario scenario identifier
|
||||||
* @param resourcePath bundled dictionary resource path
|
* @param resourcePath bundled dictionary resource path
|
||||||
* @throws IOException if reading or writing fails
|
* @throws IOException if reading or writing fails
|
||||||
*/
|
*/
|
||||||
@ParameterizedTest(name = "[{index}] {0}")
|
@ParameterizedTest(name = "[{index}] {0}")
|
||||||
@MethodSource("org.egothor.stemmer.CompileIntegrationTest#bundledDictionaryCases")
|
@MethodSource("org.egothor.stemmer.CompileIntegrationTest#bundledDictionaryCases")
|
||||||
|
@Tag("slow")
|
||||||
@DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics")
|
@DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics")
|
||||||
void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario,
|
void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario,
|
||||||
final String resourcePath) throws IOException {
|
final String resourcePath) throws IOException {
|
||||||
|
|||||||
@@ -66,7 +66,11 @@ import org.junit.jupiter.api.io.TempDir;
|
|||||||
* {@link System#exit(int)}.
|
* {@link System#exit(int)}.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("integration")
|
||||||
|
@Tag("cli")
|
||||||
|
@Tag("compile")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("construction")
|
||||||
@DisplayName("Compile")
|
@DisplayName("Compile")
|
||||||
class CompileTest {
|
class CompileTest {
|
||||||
|
|
||||||
@@ -175,6 +179,11 @@ class CompileTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("argument validation")
|
@DisplayName("argument validation")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("cli")
|
||||||
|
@Tag("compile")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
class ArgumentValidationTest {
|
class ArgumentValidationTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -70,10 +70,11 @@ import org.junit.jupiter.params.provider.MethodSource;
|
|||||||
* <li>compressed artifact reproducibility within the active format version</li>
|
* <li>compressed artifact reproducibility within the active format version</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("compat")
|
||||||
@Tag("regression")
|
@Tag("regression")
|
||||||
@Tag("determinism")
|
@Tag("determinism")
|
||||||
@Tag("serialization")
|
@Tag("serialization")
|
||||||
|
@Tag("trie")
|
||||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||||
final class CompiledTrieArtifactRegressionTest {
|
final class CompiledTrieArtifactRegressionTest {
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link DiacriticStripper}.
|
* Unit tests for {@link DiacriticStripper}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("diacritics")
|
@Tag("diacritic")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("normalization")
|
||||||
@DisplayName("DiacriticStripper")
|
@DisplayName("DiacriticStripper")
|
||||||
class DiacriticStripperTest {
|
class DiacriticStripperTest {
|
||||||
|
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ import org.junit.jupiter.api.Test;
|
|||||||
*/
|
*/
|
||||||
@DisplayName("FrequencyTrieBuilders")
|
@DisplayName("FrequencyTrieBuilders")
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("builder")
|
@Tag("construction")
|
||||||
@Tag("frequency-trie")
|
@Tag("frequency-trie")
|
||||||
class FrequencyTrieBuildersTest {
|
class FrequencyTrieBuildersTest {
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ import java.util.List;
|
|||||||
import net.jqwik.api.ForAll;
|
import net.jqwik.api.ForAll;
|
||||||
import net.jqwik.api.Label;
|
import net.jqwik.api.Label;
|
||||||
import net.jqwik.api.Property;
|
import net.jqwik.api.Property;
|
||||||
import net.jqwik.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Property-based tests for the compiled trie abstraction.
|
* Property-based tests for the compiled trie abstraction.
|
||||||
@@ -59,9 +59,9 @@ import net.jqwik.api.Tag;
|
|||||||
* core algorithm without overfitting to particular fixture data.
|
* core algorithm without overfitting to particular fixture data.
|
||||||
*/
|
*/
|
||||||
@Label("FrequencyTrie properties")
|
@Label("FrequencyTrie properties")
|
||||||
@Tag("unit")
|
|
||||||
@Tag("property")
|
@Tag("property")
|
||||||
@Tag("trie")
|
@Tag("trie")
|
||||||
|
@Tag("frequency-trie")
|
||||||
class FrequencyTrieProperties extends PropertyBasedTestSupport {
|
class FrequencyTrieProperties extends PropertyBasedTestSupport {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ package org.egothor.stemmer;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||||
@@ -44,6 +45,7 @@ import java.io.ByteArrayOutputStream;
|
|||||||
import java.io.DataInputStream;
|
import java.io.DataInputStream;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
@@ -62,6 +64,7 @@ import org.junit.jupiter.api.Test;
|
|||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("trie")
|
@Tag("trie")
|
||||||
@Tag("frequency-trie")
|
@Tag("frequency-trie")
|
||||||
|
@Tag("lookup")
|
||||||
@DisplayName("FrequencyTrie")
|
@DisplayName("FrequencyTrie")
|
||||||
class FrequencyTrieTest {
|
class FrequencyTrieTest {
|
||||||
|
|
||||||
@@ -379,6 +382,167 @@ class FrequencyTrieTest {
|
|||||||
assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1)));
|
assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that {@link FrequencyTrie#getEntries(String)} short-circuits to a one-item immutable list.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("getEntries returns a one-item list for single stored values")
|
||||||
|
void getEntriesReturnsSingleItemListForSingleStoredValue() {
|
||||||
|
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||||
|
|
||||||
|
builder.put("gamma", "only");
|
||||||
|
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
|
||||||
|
final List<ValueCount<String>> entries = trie.getEntries("gamma");
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(List.of(new ValueCount<String>("only", 1)), entries),
|
||||||
|
() -> assertThrows(UnsupportedOperationException.class, () -> entries.add(new ValueCount<String>("z", 1))));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that visitor lookup returns the same deterministic order and counts
|
||||||
|
* as the allocating APIs.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Visitor lookup matches getAll order and getEntries counts")
|
||||||
|
void visitorLookupMatchesGetAllOrderAndGetEntriesCounts() {
|
||||||
|
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||||
|
builder.put("house", "noun", 3);
|
||||||
|
builder.put("house", "verb", 2);
|
||||||
|
builder.put("house", "adjective", 1);
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
final List<String> values = new ArrayList<>();
|
||||||
|
final List<Integer> counts = new ArrayList<>();
|
||||||
|
final List<Integer> ranks = new ArrayList<>();
|
||||||
|
|
||||||
|
final int visited = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||||
|
values.add(value);
|
||||||
|
counts.add(count);
|
||||||
|
ranks.add(rank);
|
||||||
|
return true;
|
||||||
|
}, 10);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(3, visited),
|
||||||
|
() -> assertEquals(List.of("noun", "verb", "adjective"), values),
|
||||||
|
() -> assertEquals(List.of(3, 2, 1), counts),
|
||||||
|
() -> assertEquals(List.of(0, 1, 2), ranks));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies visitor maximum result and early-stop behavior.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Visitor lookup honors maxResults and sink early stop")
|
||||||
|
void visitorLookupHonorsMaxResultsAndSinkEarlyStop() {
|
||||||
|
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||||
|
builder.put("house", "noun", 3);
|
||||||
|
builder.put("house", "verb", 2);
|
||||||
|
builder.put("house", "adjective", 1);
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
final List<String> limited = new ArrayList<>();
|
||||||
|
final List<String> stopped = new ArrayList<>();
|
||||||
|
|
||||||
|
final int limitedCount = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||||
|
limited.add(value);
|
||||||
|
return true;
|
||||||
|
}, 2);
|
||||||
|
final int stoppedCount = trie.getAllNormalized("house", (value, count, rank) -> {
|
||||||
|
stopped.add(value);
|
||||||
|
return false;
|
||||||
|
}, 10);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(2, limitedCount),
|
||||||
|
() -> assertEquals(List.of("noun", "verb"), limited),
|
||||||
|
() -> assertEquals(1, stoppedCount),
|
||||||
|
() -> assertEquals(List.of("noun"), stopped));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies visitor zero, negative, missing, and first-result behavior.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Visitor lookup handles zero, negative, missing, and first-result cases")
|
||||||
|
void visitorLookupHandlesBoundaryCases() {
|
||||||
|
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||||
|
builder.put("house", "noun");
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
final int[] calls = new int[1];
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(0, trie.getAllNormalized("house", (value, count, rank) -> {
|
||||||
|
calls[0]++;
|
||||||
|
return true;
|
||||||
|
}, 0)),
|
||||||
|
() -> assertEquals(0, calls[0]),
|
||||||
|
() -> assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> trie.getAllNormalized("house", (value, count, rank) -> true, -1)),
|
||||||
|
() -> assertEquals(0, trie.getAllNormalized("missing", (value, count, rank) -> true, 10)),
|
||||||
|
() -> assertFalse(trie.getFirstNormalized("missing", (value, count, rank) -> true)),
|
||||||
|
() -> assertTrue(trie.getFirstNormalized("house", (value, count, rank) -> {
|
||||||
|
assertEquals("noun", value);
|
||||||
|
assertEquals(1, count);
|
||||||
|
assertEquals(0, rank);
|
||||||
|
return true;
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies visitor API argument validation.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Visitor lookup rejects null and invalid range arguments")
|
||||||
|
void visitorLookupRejectsNullAndInvalidRangeArguments() {
|
||||||
|
final FrequencyTrie<String> trie = rankedBuilder().build();
|
||||||
|
final char[] key = "house".toCharArray();
|
||||||
|
final FrequencyTrie.EntrySink<String> sink = (value, count, rank) -> true;
|
||||||
|
|
||||||
|
assertAll(() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAllNormalized((char[]) null, 0, 0, sink, 1)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAllNormalized(key, 0, key.length, null, 1)),
|
||||||
|
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||||
|
() -> trie.getAllNormalized(key, -1, key.length, sink, 1)),
|
||||||
|
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||||
|
() -> trie.getAllNormalized(key, 1, key.length, sink, 1)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAllNormalized((CharSequence) null, sink, 1)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAllNormalized("house", null, 1)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAll((CharSequence) null, sink, 1)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> trie.getAll("house", null, 1)),
|
||||||
|
() -> assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> trie.getAll("house", sink, -1)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies normalized char-array slices and metadata-aware CharSequence visitor
|
||||||
|
* lookup.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Visitor lookup supports normalized char slices and metadata-aware CharSequence keys")
|
||||||
|
void visitorLookupSupportsCharSlicesAndMetadataAwareKeys() {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||||
|
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||||
|
builder.put("house", "noun");
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
final char[] padded = "__house__".toCharArray();
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(1,
|
||||||
|
trie.getAllNormalized(padded, 2, 5, (value, count, rank) -> {
|
||||||
|
assertEquals("noun", value);
|
||||||
|
return true;
|
||||||
|
}, 10)),
|
||||||
|
() -> assertFalse(trie.getFirstNormalized("HOUSE", (value, count, rank) -> true),
|
||||||
|
"Normalized lookup must bypass metadata lowercasing."),
|
||||||
|
() -> assertTrue(trie.getFirst("HOUSE", (value, count, rank) -> {
|
||||||
|
assertEquals("noun", value);
|
||||||
|
return true;
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that equal frequencies prefer the shorter string representation.
|
* Verifies that equal frequencies prefer the shorter string representation.
|
||||||
*/
|
*/
|
||||||
@@ -755,6 +919,115 @@ class FrequencyTrieTest {
|
|||||||
.readFrom(new ByteArrayInputStream(serializedEmptyTrie), String[]::new, null)));
|
.readFrom(new ByteArrayInputStream(serializedEmptyTrie), String[]::new, null)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that reading a compiled trie with a negative max-expanded override
|
||||||
|
* smaller than -1 is rejected.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom rejects invalid maxExpandedIndex override")
|
||||||
|
void readFromRejectsInvalidMaxExpandedIndexOverride() {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC, -2));
|
||||||
|
|
||||||
|
assertEquals("maxExpandedIndex must be >= -1.", exception.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the max-expanded override controls dense lookup materialization
|
||||||
|
* while preserving lookup semantics.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom respects dense lookup max-expanded index override")
|
||||||
|
void readFromRespectsDenseLookupMaxExpandedIndexOverride() throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = rankedBuilder();
|
||||||
|
|
||||||
|
builder.put("a", "a");
|
||||||
|
builder.put("b", "b");
|
||||||
|
builder.put("c", "c");
|
||||||
|
builder.put("d", "d");
|
||||||
|
|
||||||
|
final FrequencyTrie<String> original = builder.build();
|
||||||
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||||
|
original.writeTo(outputStream, STRING_CODEC);
|
||||||
|
final byte[] serializedTrie = outputStream.toByteArray();
|
||||||
|
|
||||||
|
final FrequencyTrie<String> defaultDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
|
||||||
|
STRING_CODEC);
|
||||||
|
final FrequencyTrie<String> defaultDenseByNegative = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie),
|
||||||
|
String[]::new, STRING_CODEC, -1);
|
||||||
|
final FrequencyTrie<String> disabledDense = FrequencyTrie.readFrom(new ByteArrayInputStream(serializedTrie), String[]::new,
|
||||||
|
STRING_CODEC, 0);
|
||||||
|
|
||||||
|
assertAll(
|
||||||
|
() -> assertTrue(defaultDense.root().hasDenseLookup(),
|
||||||
|
"Default read should enable dense lookup for compact first-level edges."),
|
||||||
|
() -> assertTrue(defaultDenseByNegative.root().hasDenseLookup(),
|
||||||
|
"Negative override should use the default dense lookup span."),
|
||||||
|
() -> assertFalse(disabledDense.root().hasDenseLookup(),
|
||||||
|
"Zero override should disable dense lookup tables."),
|
||||||
|
() -> assertEquals(original.get("a"), disabledDense.get("a")),
|
||||||
|
() -> assertEquals(original.get("b"), disabledDense.get("b")),
|
||||||
|
() -> assertEquals(original.get("c"), disabledDense.get("c")),
|
||||||
|
() -> assertEquals(original.get("d"), disabledDense.get("d")),
|
||||||
|
() -> assertEquals(original.get("z"), disabledDense.get("z")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that cyclic serialized node references are rejected as invalid
|
||||||
|
* serialization.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom rejects cyclic serialized node references")
|
||||||
|
void readFromRejectsCyclicSerializedNodeReferences() {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 1, 2, 0, new NodeWriter[] {
|
||||||
|
dataOutput -> {
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeChar('b');
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
},
|
||||||
|
dataOutput -> {
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeChar('a');
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||||
|
|
||||||
|
assertTrue(exception.getMessage().contains("cyclic reference detected"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that child node references outside the valid serialized range are
|
||||||
|
* rejected.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom rejects invalid child node identifiers")
|
||||||
|
void readFromRejectsInvalidChildNodeId() {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(1);
|
||||||
|
dataOutput.writeChar('a');
|
||||||
|
dataOutput.writeInt(3);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||||
|
|
||||||
|
assertTrue(exception.getMessage().contains("Invalid child node id"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that deserialization rejects an invalid stream magic header.
|
* Verifies that deserialization rejects an invalid stream magic header.
|
||||||
*/
|
*/
|
||||||
@@ -785,6 +1058,27 @@ class FrequencyTrieTest {
|
|||||||
assertTrue(exception.getMessage().contains("Unsupported trie stream version"));
|
assertTrue(exception.getMessage().contains("Unsupported trie stream version"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that the latest stream version validates textual metadata blocks.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom rejects invalid textual metadata block")
|
||||||
|
void readFromRejectsInvalidTextualMetadataBlock() {
|
||||||
|
final int version = FrequencyTrie.currentFormatVersion();
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, version, 1, 0, dataOutput -> {
|
||||||
|
dataOutput.writeUTF("not valid metadata");
|
||||||
|
}, new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||||
|
|
||||||
|
assertTrue(exception.getMessage().contains("Invalid metadata block"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that deserialization rejects a negative node count.
|
* Verifies that deserialization rejects a negative node count.
|
||||||
*/
|
*/
|
||||||
@@ -862,6 +1156,129 @@ class FrequencyTrieTest {
|
|||||||
assertTrue(exception.getMessage().contains("Non-positive stored count"));
|
assertTrue(exception.getMessage().contains("Non-positive stored count"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that legacy version 1 metadata uses compatibility defaults.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom supports legacy version 1 metadata")
|
||||||
|
void readFromSupportsLegacyVersionOneMetadata() throws IOException {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 1, 1, 0, new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||||
|
|
||||||
|
assertEquals(TrieMetadata.legacy(1, WordTraversalDirection.BACKWARD), trie.metadata());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that legacy version 2 metadata stores traversal direction and uses
|
||||||
|
* compatibility defaults for other values.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom supports legacy version 2 metadata")
|
||||||
|
void readFromSupportsLegacyVersionTwoMetadata() throws IOException {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
|
||||||
|
dataOutput -> dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal()), new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||||
|
|
||||||
|
assertEquals(TrieMetadata.legacy(2, WordTraversalDirection.FORWARD), trie.metadata());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that version 3 metadata includes reduction and diacritic
|
||||||
|
* processing settings.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom parses version 3 metadata")
|
||||||
|
void readFromParsesVersionThreeMetadata() throws IOException {
|
||||||
|
final ReductionSettings reductionSettings = new ReductionSettings(
|
||||||
|
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS, 81, 4);
|
||||||
|
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 3, 1, 0,
|
||||||
|
dataOutput -> {
|
||||||
|
dataOutput.writeInt(WordTraversalDirection.BACKWARD.ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
|
||||||
|
dataOutput.writeInt(DiacriticProcessingMode.REMOVE.ordinal());
|
||||||
|
},
|
||||||
|
new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||||
|
final TrieMetadata metadata = trie.metadata();
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(3, metadata.formatVersion()),
|
||||||
|
() -> assertEquals(WordTraversalDirection.BACKWARD, metadata.traversalDirection()),
|
||||||
|
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
|
||||||
|
() -> assertEquals(DiacriticProcessingMode.REMOVE, metadata.diacriticProcessingMode()),
|
||||||
|
() -> assertEquals(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, metadata.caseProcessingMode()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that version 4 metadata additionally stores case-processing mode.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom parses version 4 case processing metadata")
|
||||||
|
void readFromParsesVersionFourCaseMetadata() throws IOException {
|
||||||
|
final ReductionSettings reductionSettings = new ReductionSettings(
|
||||||
|
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, 75, 3);
|
||||||
|
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 4, 1, 0,
|
||||||
|
dataOutput -> {
|
||||||
|
dataOutput.writeInt(WordTraversalDirection.FORWARD.ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.reductionMode().ordinal());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerMinPercent());
|
||||||
|
dataOutput.writeInt(reductionSettings.dominantWinnerOverSecondRatio());
|
||||||
|
dataOutput.writeInt(DiacriticProcessingMode.AS_IS.ordinal());
|
||||||
|
dataOutput.writeInt(CaseProcessingMode.AS_IS.ordinal());
|
||||||
|
},
|
||||||
|
new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final FrequencyTrie<String> trie = FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC);
|
||||||
|
final TrieMetadata metadata = trie.metadata();
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(4, metadata.formatVersion()),
|
||||||
|
() -> assertEquals(WordTraversalDirection.FORWARD, metadata.traversalDirection()),
|
||||||
|
() -> assertEquals(reductionSettings, metadata.reductionSettings()),
|
||||||
|
() -> assertEquals(DiacriticProcessingMode.AS_IS, metadata.diacriticProcessingMode()),
|
||||||
|
() -> assertEquals(CaseProcessingMode.AS_IS, metadata.caseProcessingMode()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that invalid legacy metadata ordinals are rejected by validation.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@Tag("persistence")
|
||||||
|
@DisplayName("readFrom rejects invalid metadata ordinal in legacy stream")
|
||||||
|
void readFromRejectsInvalidLegacyMetadataOrdinal() {
|
||||||
|
final byte[] bytes = createSerializedStream(0x45475452, 2, 1, 0,
|
||||||
|
dataOutput -> dataOutput.writeInt(999), new NodeWriter[] { dataOutput -> {
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
dataOutput.writeInt(0);
|
||||||
|
} });
|
||||||
|
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> FrequencyTrie.readFrom(new ByteArrayInputStream(bytes), String[]::new, STRING_CODEC));
|
||||||
|
|
||||||
|
assertTrue(exception.getMessage().contains("Invalid traversal direction ordinal"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes one node body into a synthetic serialized trie stream.
|
* Writes one node body into a synthetic serialized trie stream.
|
||||||
*/
|
*/
|
||||||
@@ -889,6 +1306,24 @@ class FrequencyTrieTest {
|
|||||||
*/
|
*/
|
||||||
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
|
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
|
||||||
final int rootNodeId, final NodeWriter[] nodes) {
|
final int rootNodeId, final NodeWriter[] nodes) {
|
||||||
|
return createSerializedStream(magic, version, nodeCount, rootNodeId, dataOutput -> {
|
||||||
|
// legacy and text-based versions write their metadata differently.
|
||||||
|
}, nodes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes a synthetic serialized trie stream with a metadata writer hook.
|
||||||
|
*
|
||||||
|
* @param magic stream magic
|
||||||
|
* @param version stream version
|
||||||
|
* @param nodeCount declared node count
|
||||||
|
* @param rootNodeId declared root node identifier
|
||||||
|
* @param metadata version-specific metadata writer
|
||||||
|
* @param nodes node body writers
|
||||||
|
* @return serialized bytes
|
||||||
|
*/
|
||||||
|
private static byte[] createSerializedStream(final int magic, final int version, final int nodeCount,
|
||||||
|
final int rootNodeId, final MetadataWriter metadata, final NodeWriter[] nodes) {
|
||||||
try {
|
try {
|
||||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||||
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
|
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
|
||||||
@@ -897,6 +1332,7 @@ class FrequencyTrieTest {
|
|||||||
dataOutputStream.writeInt(version);
|
dataOutputStream.writeInt(version);
|
||||||
dataOutputStream.writeInt(nodeCount);
|
dataOutputStream.writeInt(nodeCount);
|
||||||
dataOutputStream.writeInt(rootNodeId);
|
dataOutputStream.writeInt(rootNodeId);
|
||||||
|
metadata.write(dataOutputStream);
|
||||||
|
|
||||||
for (NodeWriter node : nodes) {
|
for (NodeWriter node : nodes) {
|
||||||
node.write(dataOutputStream);
|
node.write(dataOutputStream);
|
||||||
@@ -908,4 +1344,19 @@ class FrequencyTrieTest {
|
|||||||
throw new IllegalStateException("Unexpected I/O while building synthetic trie stream.", exception);
|
throw new IllegalStateException("Unexpected I/O while building synthetic trie stream.", exception);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes one synthetic metadata block.
|
||||||
|
*/
|
||||||
|
@FunctionalInterface
|
||||||
|
private interface MetadataWriter {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes metadata bytes for one stream version.
|
||||||
|
*
|
||||||
|
* @param dataOutput output stream
|
||||||
|
* @throws IOException if writing fails
|
||||||
|
*/
|
||||||
|
void write(DataOutputStream dataOutput) throws IOException;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,10 +65,10 @@ import org.junit.jupiter.api.io.TempDir;
|
|||||||
* stems declared by the source dictionary.
|
* stems declared by the source dictionary.
|
||||||
*/
|
*/
|
||||||
@DisplayName("Deterministic fuzz-style trie and stemmer compilation")
|
@DisplayName("Deterministic fuzz-style trie and stemmer compilation")
|
||||||
@Tag("unit")
|
|
||||||
@Tag("fuzz")
|
@Tag("fuzz")
|
||||||
@Tag("trie")
|
@Tag("trie")
|
||||||
@Tag("stemming")
|
@Tag("stemmer")
|
||||||
|
@Tag("determinism")
|
||||||
class FuzzStemmerAndTrieCompilationTest {
|
class FuzzStemmerAndTrieCompilationTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|||||||
import net.jqwik.api.ForAll;
|
import net.jqwik.api.ForAll;
|
||||||
import net.jqwik.api.Label;
|
import net.jqwik.api.Label;
|
||||||
import net.jqwik.api.Property;
|
import net.jqwik.api.Property;
|
||||||
import net.jqwik.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Property-based tests for {@link PatchCommandEncoder}.
|
* Property-based tests for {@link PatchCommandEncoder}.
|
||||||
@@ -47,9 +47,9 @@ import net.jqwik.api.Tag;
|
|||||||
* reconstruct the exact requested target.
|
* reconstruct the exact requested target.
|
||||||
*/
|
*/
|
||||||
@Label("PatchCommandEncoder properties")
|
@Label("PatchCommandEncoder properties")
|
||||||
@Tag("unit")
|
|
||||||
@Tag("property")
|
@Tag("property")
|
||||||
@Tag("patch")
|
@Tag("patch")
|
||||||
|
@Tag("stemmer")
|
||||||
class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
package org.egothor.stemmer;
|
package org.egothor.stemmer;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
@@ -68,6 +69,8 @@ import org.junit.jupiter.params.provider.MethodSource;
|
|||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("stemmer")
|
@Tag("stemmer")
|
||||||
@Tag("patch")
|
@Tag("patch")
|
||||||
|
@Tag("encoding")
|
||||||
|
@Tag("apply")
|
||||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||||
class PatchCommandEncoderTest {
|
class PatchCommandEncoderTest {
|
||||||
|
|
||||||
@@ -147,6 +150,63 @@ class PatchCommandEncoderTest {
|
|||||||
Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher"));
|
Arguments.of(10, "teacher", PatchCommandEncoder.NOOP_PATCH, "teacher"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides explicit forward-direction single-instruction patch application cases.
|
||||||
|
*
|
||||||
|
* @return test arguments
|
||||||
|
*/
|
||||||
|
private static Stream<Arguments> provideForwardSingleInstructionApplyCases() {
|
||||||
|
return Stream.of(
|
||||||
|
// 1
|
||||||
|
Arguments.of(1, "abcd", "Db", "cd"),
|
||||||
|
// 2
|
||||||
|
Arguments.of(2, "abc", "Ia", "aabc"),
|
||||||
|
// 3
|
||||||
|
Arguments.of(3, "abc", "Ra", "abc"),
|
||||||
|
// 4
|
||||||
|
Arguments.of(4, "abc", "-a", "abc"),
|
||||||
|
// 5
|
||||||
|
Arguments.of(5, "abc", PatchCommandEncoder.NOOP_PATCH, "abc"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides forward-direction applyTo cases that exercise preserve-only and
|
||||||
|
* non-preserve branches.
|
||||||
|
*
|
||||||
|
* @return test arguments
|
||||||
|
*/
|
||||||
|
private static Stream<Arguments> provideForwardApplyToCases() {
|
||||||
|
return Stream.of(
|
||||||
|
// 1
|
||||||
|
Arguments.of(1, "book", "-aRa", "baok"),
|
||||||
|
// 2
|
||||||
|
Arguments.of(2, "abc", "-dRa", "abc"),
|
||||||
|
// 3
|
||||||
|
Arguments.of(3, "abc", "DdRa", "abc"),
|
||||||
|
// 4
|
||||||
|
Arguments.of(4, "abc", "-dIa", "abc"),
|
||||||
|
// 5
|
||||||
|
Arguments.of(5, "abc", "Na-a", "abc"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides empty-source forward applyTo cases that cover empty-source
|
||||||
|
* instruction handling.
|
||||||
|
*
|
||||||
|
* @return test arguments
|
||||||
|
*/
|
||||||
|
private static Stream<Arguments> provideForwardEmptySourceApplyCases() {
|
||||||
|
return Stream.of(
|
||||||
|
// 1
|
||||||
|
Arguments.of(1, "IaIb", "ab"),
|
||||||
|
// 2
|
||||||
|
Arguments.of(2, "-aRa", ""),
|
||||||
|
// 3
|
||||||
|
Arguments.of(3, "IaRa", ""),
|
||||||
|
// 4
|
||||||
|
Arguments.of(4, "Na-a", ""));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Provides malformed or index-invalid patch inputs that must preserve the
|
* Provides malformed or index-invalid patch inputs that must preserve the
|
||||||
* original source according to the implementation contract.
|
* original source according to the implementation contract.
|
||||||
@@ -236,12 +296,31 @@ class PatchCommandEncoderTest {
|
|||||||
return new StringBuilder(text).reverse().toString();
|
return new StringBuilder(text).reverse().toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies a patch into a right-sized output buffer and returns the produced
|
||||||
|
* string.
|
||||||
|
*
|
||||||
|
* @param source source text
|
||||||
|
* @param patch patch command
|
||||||
|
* @param traversalDirection traversal direction
|
||||||
|
* @return transformed text
|
||||||
|
*/
|
||||||
|
private static String applyToString(final String source, final String patch,
|
||||||
|
final WordTraversalDirection traversalDirection) {
|
||||||
|
final char[] output = new char[Math.max(source.length() + 16, 16)];
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(source, patch, traversalDirection, output, 0, output.length);
|
||||||
|
return new String(output, 0, produced);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests constructor validation and basic instantiation behavior.
|
* Tests constructor validation and basic instantiation behavior.
|
||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("construction")
|
@DisplayName("construction")
|
||||||
@Tag("constructor")
|
@Tag("construction")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("patch")
|
||||||
class ConstructionTests {
|
class ConstructionTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -326,7 +405,10 @@ class PatchCommandEncoderTest {
|
|||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("encode(String, String)")
|
@DisplayName("encode(String, String)")
|
||||||
@Tag("encode")
|
@Tag("encoding")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("patch")
|
||||||
class EncodeTests {
|
class EncodeTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -461,6 +543,9 @@ class PatchCommandEncoderTest {
|
|||||||
@Nested
|
@Nested
|
||||||
@DisplayName("apply(String, String)")
|
@DisplayName("apply(String, String)")
|
||||||
@Tag("apply")
|
@Tag("apply")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("patch")
|
||||||
class ApplyTests {
|
class ApplyTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -535,6 +620,101 @@ class PatchCommandEncoderTest {
|
|||||||
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
|
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies explicit single-instruction forward patch application
|
||||||
|
* semantics.
|
||||||
|
*
|
||||||
|
* @param caseId numeric case identifier
|
||||||
|
* @param source source word
|
||||||
|
* @param patch encoded patch command
|
||||||
|
* @param expected expected transformed word
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "[{index}] case {0}: forward single instruction apply({1}, {2}) -> {3}")
|
||||||
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardSingleInstructionApplyCases")
|
||||||
|
@DisplayName("applies forward single instruction patches correctly")
|
||||||
|
void shouldApplyForwardSingleInstructionsExplicitly(int caseId, String source, String patch, String expected) {
|
||||||
|
assertEquals(expected, PatchCommandEncoder.apply(source, patch, WordTraversalDirection.FORWARD),
|
||||||
|
() -> "Case " + caseId + " failed for source='" + source + "', patch='" + patch + "'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies forward single-instruction malformed commands fail fast.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("throws for unsupported forward opcode and NOOP argument")
|
||||||
|
void shouldThrowForUnsupportedForwardOpcodeAndNoopArgument() {
|
||||||
|
assertAll(() -> assertEquals("Unsupported patch opcode: X",
|
||||||
|
assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.apply("abc", "Xa", WordTraversalDirection.FORWARD))
|
||||||
|
.getMessage()),
|
||||||
|
() -> assertEquals("Unsupported NOOP patch argument: `",
|
||||||
|
assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.apply("abc", "N`", WordTraversalDirection.FORWARD))
|
||||||
|
.getMessage()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies explicit forward-applyTo cases that exercise preserve-only and
|
||||||
|
* non-preserve branches.
|
||||||
|
*
|
||||||
|
* @param caseId numeric case identifier
|
||||||
|
* @param source source word
|
||||||
|
* @param patch encoded patch command
|
||||||
|
* @param expected expected transformed word
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "[{index}] case {0}: applyToForward({1}, {2}) -> {3}")
|
||||||
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardApplyToCases")
|
||||||
|
@DisplayName("applyTo handles forward preserve-only and mutation branches")
|
||||||
|
void shouldApplyToForwardPreserveAndMutationBranches(int caseId, String source, String patch, String expected) {
|
||||||
|
final char[] output = new char[Math.max(source.length() + 16, 16)];
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.FORWARD, output, 0,
|
||||||
|
output.length);
|
||||||
|
|
||||||
|
assertAll(
|
||||||
|
() -> assertEquals(expected.length(), produced,
|
||||||
|
() -> "Case " + caseId + " produced wrong length."),
|
||||||
|
() -> assertEquals(expected, new String(output, 0, produced),
|
||||||
|
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies empty-source forward applyTo behavior for insert-only and malformed
|
||||||
|
* instructions.
|
||||||
|
*
|
||||||
|
* @param caseId numeric case identifier
|
||||||
|
* @param patch encoded patch command
|
||||||
|
* @param expected expected transformed word
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "[{index}] case {0}: applyToForward(\"\", {1}) -> \"{2}\"")
|
||||||
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideForwardEmptySourceApplyCases")
|
||||||
|
@DisplayName("applies forward empty-source patches correctly")
|
||||||
|
void shouldApplyToForwardEmptySourceCases(int caseId, String patch, String expected) {
|
||||||
|
final char[] output = new char[Math.max(expected.length() + 16, 16)];
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo("", patch, WordTraversalDirection.FORWARD, output, 0,
|
||||||
|
output.length);
|
||||||
|
|
||||||
|
assertAll(
|
||||||
|
() -> assertEquals(expected.length(), produced,
|
||||||
|
() -> "Case " + caseId + " produced wrong length."),
|
||||||
|
() -> assertEquals(expected, new String(output, 0, produced),
|
||||||
|
() -> "Case " + caseId + " failed for patch='" + patch + "'."));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies malformed empty-source forward patches fail fast and preserve
|
||||||
|
* empty-source semantics.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("throws for unsupported NOOP argument on empty-source forward patch")
|
||||||
|
void shouldThrowForUnsupportedNoopArgumentOnForwardEmptySource() {
|
||||||
|
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.apply("", "N`Ra", WordTraversalDirection.FORWARD));
|
||||||
|
|
||||||
|
assertEquals("Unsupported NOOP patch argument: `", exception.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies explicit patch application cases.
|
* Verifies explicit patch application cases.
|
||||||
*
|
*
|
||||||
@@ -590,6 +770,181 @@ class PatchCommandEncoderTest {
|
|||||||
assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId
|
assertEquals(source, PatchCommandEncoder.apply(source, malformedPatch), () -> "Case " + caseId
|
||||||
+ " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'.");
|
+ " failed for source='" + source + "', malformedPatch='" + malformedPatch + "'.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies buffer application against string-returning application.
|
||||||
|
*
|
||||||
|
* @param caseId numeric case identifier
|
||||||
|
* @param source source word
|
||||||
|
* @param patch patch command
|
||||||
|
* @param expected expected transformed word
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "[{index}] case {0}: applyTo({1}, {2}) -> {3}")
|
||||||
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideApplyCases")
|
||||||
|
@DisplayName("applyTo matches apply for explicit backward patch commands")
|
||||||
|
void shouldApplyToBufferLikeApplyForBackwardCommands(int caseId, String source, String patch, String expected) {
|
||||||
|
final char[] output = "___..............".toCharArray();
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(source, patch, WordTraversalDirection.BACKWARD, output, 3,
|
||||||
|
output.length - 3);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(expected.length(), produced, () -> "Case " + caseId + " produced wrong length."),
|
||||||
|
() -> assertEquals(expected, new String(output, 3, produced)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies char-array source slices.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo supports char-array source slices")
|
||||||
|
void shouldApplyToCharArraySourceSlice() {
|
||||||
|
final char[] source = "__teacher__".toCharArray();
|
||||||
|
final char[] output = new char[16];
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(source, 2, 7, "Db", WordTraversalDirection.BACKWARD,
|
||||||
|
output, 1, output.length - 1);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(5, produced), () -> assertEquals("teach", new String(output, 1, produced)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies null and range validation for buffer application.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo rejects null and invalid range arguments")
|
||||||
|
void shouldRejectNullAndInvalidRangeArguments() {
|
||||||
|
final char[] source = "teacher".toCharArray();
|
||||||
|
final char[] output = new char[16];
|
||||||
|
|
||||||
|
assertAll(() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo((CharSequence) null, "Db", WordTraversalDirection.BACKWARD,
|
||||||
|
output, 0, output.length)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo("teacher", "Db", null, output, 0, output.length)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD, null,
|
||||||
|
0, output.length)),
|
||||||
|
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo("teacher", "Db", WordTraversalDirection.BACKWARD,
|
||||||
|
output, -1, output.length)),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo((char[]) null, 0, 7, "Db",
|
||||||
|
WordTraversalDirection.BACKWARD, output, 0, output.length)),
|
||||||
|
() -> assertThrows(IndexOutOfBoundsException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo(source, 1, source.length, "Db",
|
||||||
|
WordTraversalDirection.BACKWARD, output, 0, output.length)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies null, empty, and canonical NOOP patch preservation.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo preserves source for null, empty, and canonical NOOP patches")
|
||||||
|
void shouldApplyToPreserveSourceForEmptyCompatibilityPatches() {
|
||||||
|
final char[] nullPatchOutput = new char[8];
|
||||||
|
final char[] emptyPatchOutput = new char[8];
|
||||||
|
final char[] noopOutput = new char[8];
|
||||||
|
|
||||||
|
final int nullPatchLength = PatchCommandEncoder.applyTo("teacher", null, WordTraversalDirection.BACKWARD,
|
||||||
|
nullPatchOutput, 0, nullPatchOutput.length);
|
||||||
|
final int emptyPatchLength = PatchCommandEncoder.applyTo("teacher", "", WordTraversalDirection.BACKWARD,
|
||||||
|
emptyPatchOutput, 0, emptyPatchOutput.length);
|
||||||
|
final int noopLength = PatchCommandEncoder.applyTo("teacher", PatchCommandEncoder.NOOP_PATCH,
|
||||||
|
WordTraversalDirection.BACKWARD, noopOutput, 0, noopOutput.length);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(7, nullPatchLength),
|
||||||
|
() -> assertEquals("teacher", new String(nullPatchOutput, 0, nullPatchLength)),
|
||||||
|
() -> assertEquals(7, emptyPatchLength),
|
||||||
|
() -> assertEquals("teacher", new String(emptyPatchOutput, 0, emptyPatchLength)),
|
||||||
|
() -> assertEquals(7, noopLength),
|
||||||
|
() -> assertEquals("teacher", new String(noopOutput, 0, noopLength)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies insufficient capacity behavior.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo reports insufficient capacity without writing output")
|
||||||
|
void shouldReportInsufficientCapacityWithoutWritingOutput() {
|
||||||
|
final char[] output = "xxxx".toCharArray();
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo("abc", "Ic", WordTraversalDirection.BACKWARD, output, 0,
|
||||||
|
output.length - 1);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(PatchCommandEncoder.APPLY_INSUFFICIENT_CAPACITY, produced),
|
||||||
|
() -> assertArrayEquals("xxxx".toCharArray(), output));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies exception parity with string-returning application.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo throws for unsupported opcode and NOOP argument")
|
||||||
|
void shouldApplyToThrowForUnsupportedOpcodeAndNoopArgument() {
|
||||||
|
final char[] output = new char[8];
|
||||||
|
|
||||||
|
assertAll(() -> {
|
||||||
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo("abc", "Xa", WordTraversalDirection.BACKWARD, output, 0,
|
||||||
|
output.length));
|
||||||
|
assertEquals("Unsupported patch opcode: X", exception.getMessage());
|
||||||
|
}, () -> {
|
||||||
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo("abc", "Nb", WordTraversalDirection.BACKWARD, output, 0,
|
||||||
|
output.length));
|
||||||
|
assertEquals("Unsupported NOOP patch argument: b", exception.getMessage());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies malformed compatibility behavior for buffer application.
|
||||||
|
*
|
||||||
|
* @param caseId numeric case identifier
|
||||||
|
* @param source original source
|
||||||
|
* @param malformedPatch malformed patch
|
||||||
|
*/
|
||||||
|
@ParameterizedTest(name = "[{index}] case {0}: malformed applyTo patch {2} preserves {1}")
|
||||||
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideMalformedPatchCases")
|
||||||
|
@DisplayName("applyTo preserves source for malformed or index-invalid patch commands")
|
||||||
|
void shouldApplyToPreserveSourceForMalformedOrIndexInvalidPatchCommands(int caseId, String source,
|
||||||
|
String malformedPatch) {
|
||||||
|
final char[] output = new char[Math.max(source.length(), 1)];
|
||||||
|
|
||||||
|
final int produced = PatchCommandEncoder.applyTo(source, malformedPatch, WordTraversalDirection.BACKWARD,
|
||||||
|
output, 0, output.length);
|
||||||
|
|
||||||
|
assertAll(() -> assertEquals(source.length(), produced, () -> "Case " + caseId + " produced wrong length."),
|
||||||
|
() -> assertEquals(source, new String(output, 0, produced)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies explicit traversal direction for buffer application.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo follows explicit forward traversal direction")
|
||||||
|
void shouldApplyToWithForwardTraversalDirection() {
|
||||||
|
final PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||||
|
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||||
|
.build();
|
||||||
|
final String patch = encoder.encode("cities", "city");
|
||||||
|
|
||||||
|
assertEquals(PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD),
|
||||||
|
applyToString("cities", patch, WordTraversalDirection.FORWARD));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies overlapping source/output slices are rejected.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("applyTo rejects overlapping char-array source and output ranges")
|
||||||
|
void shouldRejectOverlappingSourceAndOutputRanges() {
|
||||||
|
final char[] buffer = "teacher....".toCharArray();
|
||||||
|
|
||||||
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> PatchCommandEncoder.applyTo(buffer, 0, 7, "Db", WordTraversalDirection.BACKWARD, buffer, 2, 5));
|
||||||
|
|
||||||
|
assertEquals("source and output ranges must not overlap.", exception.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -598,6 +953,9 @@ class PatchCommandEncoderTest {
|
|||||||
@Nested
|
@Nested
|
||||||
@DisplayName("stemming-oriented scenarios")
|
@DisplayName("stemming-oriented scenarios")
|
||||||
@Tag("regression")
|
@Tag("regression")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("patch")
|
||||||
class StemmingScenarioTests {
|
class StemmingScenarioTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -658,7 +1016,10 @@ class PatchCommandEncoderTest {
|
|||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("reversed-word processing")
|
@DisplayName("reversed-word processing")
|
||||||
@Tag("reverse")
|
@Tag("normalization")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("patch")
|
||||||
class ReversedWordProcessingTests {
|
class ReversedWordProcessingTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -743,6 +1104,7 @@ class PatchCommandEncoderTest {
|
|||||||
*/
|
*/
|
||||||
@ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}")
|
@ParameterizedTest(name = "[{index}] case {0}: mirrored consistency for {1} -> {2}")
|
||||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||||
|
@Tag("normalization")
|
||||||
@DisplayName("preserves correctness under mirrored input orientation")
|
@DisplayName("preserves correctness under mirrored input orientation")
|
||||||
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
|
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
|
||||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||||
|
|||||||
@@ -75,6 +75,8 @@ import org.junit.jupiter.api.io.TempDir;
|
|||||||
@DisplayName("StemmerDictionaryParser")
|
@DisplayName("StemmerDictionaryParser")
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("parser")
|
@Tag("parser")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
class StemmerDictionaryParserTest {
|
class StemmerDictionaryParserTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -97,6 +99,10 @@ class StemmerDictionaryParserTest {
|
|||||||
/**
|
/**
|
||||||
* Log handler capturing parser diagnostics for assertions.
|
* Log handler capturing parser diagnostics for assertions.
|
||||||
*/
|
*/
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("parser")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
private static final class CapturedLogHandler extends Handler {
|
private static final class CapturedLogHandler extends Handler {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -157,6 +163,10 @@ class StemmerDictionaryParserTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("parse(Reader, String, EntryHandler)")
|
@DisplayName("parse(Reader, String, EntryHandler)")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("parser")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
class ReaderParsingTests {
|
class ReaderParsingTests {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -333,6 +343,10 @@ class StemmerDictionaryParserTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("parse(Path, EntryHandler) and parse(String, EntryHandler)")
|
@DisplayName("parse(Path, EntryHandler) and parse(String, EntryHandler)")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("parser")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
class FileParsingTests {
|
class FileParsingTests {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -423,6 +437,10 @@ class StemmerDictionaryParserTest {
|
|||||||
|
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("ParseStatistics")
|
@DisplayName("ParseStatistics")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("parser")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
class ParseStatisticsTests {
|
class ParseStatisticsTests {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -54,9 +54,10 @@ import org.junit.jupiter.api.io.TempDir;
|
|||||||
/**
|
/**
|
||||||
* Tests for {@link StemmerKnowledgeExperiment}.
|
* Tests for {@link StemmerKnowledgeExperiment}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
|
||||||
@Tag("integration")
|
@Tag("integration")
|
||||||
@Tag("stemmer")
|
@Tag("stemmer")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
final class StemmerKnowledgeExperimentTest {
|
final class StemmerKnowledgeExperimentTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ import static org.junit.jupiter.api.Assertions.assertSame;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.mockStatic;
|
import static org.mockito.Mockito.mockStatic;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
@@ -91,6 +93,8 @@ import org.mockito.MockedStatic;
|
|||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("io")
|
@Tag("io")
|
||||||
@Tag("persistence")
|
@Tag("persistence")
|
||||||
|
@Tag("serialization")
|
||||||
|
@Tag("trie")
|
||||||
@DisplayName("StemmerPatchTrieBinaryIO")
|
@DisplayName("StemmerPatchTrieBinaryIO")
|
||||||
class StemmerPatchTrieBinaryIOTest {
|
class StemmerPatchTrieBinaryIOTest {
|
||||||
|
|
||||||
@@ -130,6 +134,10 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("write(...)")
|
@DisplayName("write(...)")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("persistence")
|
||||||
class WriteTests {
|
class WriteTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -286,6 +294,10 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("read(...)")
|
@DisplayName("read(...)")
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("persistence")
|
||||||
class ReadTests {
|
class ReadTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -299,9 +311,19 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
"read(Path) must reject null path."),
|
"read(Path) must reject null path."),
|
||||||
() -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((String) null),
|
() -> assertThrows(NullPointerException.class, () -> StemmerPatchTrieBinaryIO.read((String) null),
|
||||||
"read(String) must reject null file name."),
|
"read(String) must reject null file name."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.read((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
|
"read(Path, int) must reject null path."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.read((String) null,
|
||||||
|
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
|
"read(String, int) must reject null file name."),
|
||||||
() -> assertThrows(NullPointerException.class,
|
() -> assertThrows(NullPointerException.class,
|
||||||
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null),
|
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null),
|
||||||
"read(InputStream) must reject null input stream."));
|
"read(InputStream) must reject null input stream."),
|
||||||
|
() -> assertThrows(NullPointerException.class,
|
||||||
|
() -> StemmerPatchTrieBinaryIO.read((ByteArrayInputStream) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
|
"read(InputStream, int) must reject null input stream."));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -385,6 +407,143 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that stream overload with dense span override delegates to the
|
||||||
|
* four-argument readFrom method.
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should delegate stream read with dense span override")
|
||||||
|
void shouldDelegateInputStreamReadWithDenseSpanOverride() throws IOException {
|
||||||
|
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||||
|
final byte[] gzipPayload = gzip("binary-content-with-max-expanded-index");
|
||||||
|
|
||||||
|
try (@SuppressWarnings("rawtypes")
|
||||||
|
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||||
|
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||||
|
|
||||||
|
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO
|
||||||
|
.read(new ByteArrayInputStream(gzipPayload), 17);
|
||||||
|
|
||||||
|
assertSame(expectedTrie, actualTrie,
|
||||||
|
"read(InputStream, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||||
|
|
||||||
|
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), eq(17)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that path overload with dense span override delegates to the
|
||||||
|
* same method overload with the override parameter.
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should delegate path read with dense span override")
|
||||||
|
void shouldDelegatePathReadWithDenseSpanOverride() throws IOException {
|
||||||
|
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||||
|
final Path sourceFile = temporaryDirectory.resolve("input-max-expanded.bin.gz");
|
||||||
|
Files.write(sourceFile, gzip("path-based-max-expanded-index"));
|
||||||
|
|
||||||
|
try (@SuppressWarnings("rawtypes")
|
||||||
|
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||||
|
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||||
|
|
||||||
|
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile, 0);
|
||||||
|
|
||||||
|
assertSame(expectedTrie, actualTrie,
|
||||||
|
"read(Path, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||||
|
|
||||||
|
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), eq(0)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that string path overload with dense span override delegates to the
|
||||||
|
* same method overload with the override parameter.
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should delegate file name read with dense span override")
|
||||||
|
void shouldDelegateStringReadWithDenseSpanOverride() throws IOException {
|
||||||
|
final FrequencyTrie<String> expectedTrie = mock(FrequencyTrie.class);
|
||||||
|
final Path sourceFile = temporaryDirectory.resolve("input-string-max-expanded.bin.gz");
|
||||||
|
Files.write(sourceFile, gzip("string-based-max-expanded-index"));
|
||||||
|
|
||||||
|
try (@SuppressWarnings("rawtypes")
|
||||||
|
MockedStatic<FrequencyTrie> mockedStatic = mockStatic(FrequencyTrie.class)) {
|
||||||
|
mockedStatic.when(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), anyInt())).thenReturn(expectedTrie);
|
||||||
|
|
||||||
|
final FrequencyTrie<String> actualTrie = StemmerPatchTrieBinaryIO.read(sourceFile.toString(), 32);
|
||||||
|
|
||||||
|
assertSame(expectedTrie, actualTrie,
|
||||||
|
"read(String, int) must return the trie produced by FrequencyTrie.readFrom(...).");
|
||||||
|
|
||||||
|
mockedStatic.verify(() -> FrequencyTrie.readFrom(any(DataInputStream.class), any(),
|
||||||
|
any(FrequencyTrie.ValueStreamCodec.class), eq(32)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that metadata-only read parses and returns the persisted metadata.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should read metadata from gzip payload")
|
||||||
|
void shouldReadMetadataFromGzipPayload() throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("run", PatchCommandEncoder.builder().build().encode("running", "run"));
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
|
||||||
|
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||||
|
StemmerPatchTrieBinaryIO.write(trie, outputStream);
|
||||||
|
|
||||||
|
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(new ByteArrayInputStream(outputStream.toByteArray()));
|
||||||
|
|
||||||
|
assertEquals(trie.metadata(), metadata,
|
||||||
|
"readMetadata(InputStream) must return the same metadata persisted by write().");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that metadata can be read from a binary file path.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should read metadata from file path")
|
||||||
|
void shouldReadMetadataFromPath() throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
|
||||||
|
final Path sourceFile = temporaryDirectory.resolve("metadata-path.bin.gz");
|
||||||
|
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
|
||||||
|
|
||||||
|
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile);
|
||||||
|
assertEquals(trie.metadata(), metadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that metadata can be read from a binary file name.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Should read metadata from file name")
|
||||||
|
void shouldReadMetadataFromStringPath() throws IOException {
|
||||||
|
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<String>(String[]::new,
|
||||||
|
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||||
|
builder.put("city", PatchCommandEncoder.builder().build().encode("cities", "city"));
|
||||||
|
final FrequencyTrie<String> trie = builder.build();
|
||||||
|
|
||||||
|
final Path sourceFile = temporaryDirectory.resolve("metadata-string.bin.gz");
|
||||||
|
StemmerPatchTrieBinaryIO.write(trie, sourceFile);
|
||||||
|
|
||||||
|
final TrieMetadata metadata = StemmerPatchTrieBinaryIO.readMetadata(sourceFile.toString());
|
||||||
|
assertEquals(trie.metadata(), metadata);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that malformed non-GZip input is reported as an I/O failure.
|
* Verifies that malformed non-GZip input is reported as an I/O failure.
|
||||||
*/
|
*/
|
||||||
@@ -482,6 +641,10 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
/**
|
/**
|
||||||
* Output stream that records whether it has been closed.
|
* Output stream that records whether it has been closed.
|
||||||
*/
|
*/
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("persistence")
|
||||||
private static final class TrackingOutputStream extends ByteArrayOutputStream {
|
private static final class TrackingOutputStream extends ByteArrayOutputStream {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -508,6 +671,10 @@ class StemmerPatchTrieBinaryIOTest {
|
|||||||
/**
|
/**
|
||||||
* Input stream that records whether it has been closed.
|
* Input stream that records whether it has been closed.
|
||||||
*/
|
*/
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("persistence")
|
||||||
private static final class TrackingInputStream extends ByteArrayInputStream {
|
private static final class TrackingInputStream extends ByteArrayInputStream {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -85,9 +85,12 @@ import org.junit.jupiter.params.provider.MethodSource;
|
|||||||
* <li>the current bundled language set, including right-to-left metadata</li>
|
* <li>the current bundled language set, including right-to-left metadata</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
|
||||||
@Tag("integration")
|
@Tag("integration")
|
||||||
@Tag("stemmer")
|
@Tag("stemmer")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("parser")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("persistence")
|
||||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||||
final class StemmerPatchTrieLoaderTest {
|
final class StemmerPatchTrieLoaderTest {
|
||||||
|
|
||||||
@@ -210,36 +213,43 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
Arguments.of("14-load-binary-string",
|
Arguments.of("14-load-binary-string",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null),
|
||||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||||
Arguments.of("15-load-binary-stream",
|
Arguments.of("15-load-binary-path-override",
|
||||||
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((Path) null, FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
|
"path"),
|
||||||
|
Arguments.of("16-load-binary-string-override",
|
||||||
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null,
|
||||||
|
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX),
|
||||||
|
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||||
|
Arguments.of("17-load-binary-stream",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null),
|
||||||
"inputStream"),
|
"inputStream"),
|
||||||
Arguments.of("16-save-binary-null-trie-path",
|
Arguments.of("18-save-binary-null-trie-path",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"),
|
||||||
Arguments.of("17-save-binary-null-path",
|
Arguments.of("19-save-binary-null-path",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"),
|
||||||
Arguments.of("18-save-binary-null-trie-string",
|
Arguments.of("20-save-binary-null-trie-string",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()),
|
||||||
"trie"),
|
"trie"),
|
||||||
Arguments.of("19-save-binary-null-string",
|
Arguments.of("21-save-binary-null-string",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
|
||||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||||
Arguments.of("20-load-language-null-metadata",
|
Arguments.of("22-load-language-null-metadata",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||||
true, (TrieMetadata) null),
|
true, (TrieMetadata) null),
|
||||||
"metadata"),
|
"metadata"),
|
||||||
Arguments.of("21-load-path-null-metadata",
|
Arguments.of("23-load-path-null-metadata",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
|
||||||
"metadata"),
|
"metadata"),
|
||||||
Arguments.of("22-load-string-null-metadata",
|
Arguments.of("24-load-string-null-metadata",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||||
(TrieMetadata) null),
|
(TrieMetadata) null),
|
||||||
"metadata"),
|
"metadata"),
|
||||||
Arguments.of("23-load-binary-metadata-path-null",
|
Arguments.of("25-load-binary-metadata-path-null",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
|
||||||
Arguments.of("24-load-binary-metadata-string-null",
|
Arguments.of("26-load-binary-metadata-string-null",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
|
||||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||||
Arguments.of("25-load-binary-metadata-stream-null",
|
Arguments.of("27-load-binary-metadata-stream-null",
|
||||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
|
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
|
||||||
"inputStream"));
|
"inputStream"));
|
||||||
}
|
}
|
||||||
@@ -258,6 +268,9 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("API contracts")
|
@DisplayName("API contracts")
|
||||||
|
@Tag("validation")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("trie")
|
||||||
final class ApiContractTests {
|
final class ApiContractTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -306,11 +319,83 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Focused internal loader behavior tests.
|
||||||
|
*/
|
||||||
|
@Nested
|
||||||
|
@DisplayName("Internal helper behavior")
|
||||||
|
@Tag("construction")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("trie")
|
||||||
|
final class InternalLoaderBehaviorTests {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that bundled language loading follows explicit
|
||||||
|
* right-to-left metadata mapping.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("bundled language loading must infer traversal direction from language metadata")
|
||||||
|
void shouldLoadBundledLanguagesUsingLanguageRightToLeftMetadata() throws IOException {
|
||||||
|
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||||
|
final FrequencyTrie<String> leftToRightDictionary = StemmerPatchTrieLoader.load(
|
||||||
|
StemmerPatchTrieLoader.Language.US_UK, true, settings);
|
||||||
|
final FrequencyTrie<String> rightToLeftDictionary = StemmerPatchTrieLoader.load(
|
||||||
|
StemmerPatchTrieLoader.Language.FA_IR, true, settings);
|
||||||
|
|
||||||
|
assertEquals(WordTraversalDirection.BACKWARD, leftToRightDictionary.traversalDirection(),
|
||||||
|
"Left-to-right languages should use backward traversal.");
|
||||||
|
assertEquals(WordTraversalDirection.FORWARD, rightToLeftDictionary.traversalDirection(),
|
||||||
|
"Right-to-left languages should use forward traversal.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies the mode-based and settings-based bundled load overloads remain
|
||||||
|
* semantically consistent for traversal direction.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("load(Language,.., ReductionMode) and load(Language,.., ReductionSettings) should agree on traversal direction")
|
||||||
|
void shouldKeepBundledTraversalDirectionConsistentAcrossOverloads() throws IOException {
|
||||||
|
final FrequencyTrie<String> byMode = StemmerPatchTrieLoader.load(
|
||||||
|
StemmerPatchTrieLoader.Language.US_UK, true, DEFAULT_REDUCTION_MODE);
|
||||||
|
final FrequencyTrie<String> bySettings = StemmerPatchTrieLoader.load(
|
||||||
|
StemmerPatchTrieLoader.Language.US_UK, true,
|
||||||
|
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE));
|
||||||
|
|
||||||
|
assertEquals(byMode.traversalDirection(), bySettings.traversalDirection());
|
||||||
|
assertEquals(byMode.metadata().reductionSettings().reductionMode(),
|
||||||
|
bySettings.metadata().reductionSettings().reductionMode());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies bundled resource access succeeds for known resources and fails
|
||||||
|
* for unknown resources.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("openBundledResource should return readable streams for known resources and fail for unknown ones")
|
||||||
|
void shouldOpenBundledResourcesSuccessfullyAndRejectUnknowns() throws IOException {
|
||||||
|
final String resourcePath = StemmerPatchTrieLoader.Language.US_UK.resourcePath();
|
||||||
|
try (InputStream inputStream = StemmerPatchTrieLoader.openBundledResource(resourcePath);
|
||||||
|
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||||
|
assertNotNull(reader.readLine(), "Known bundled resource must expose readable content.");
|
||||||
|
}
|
||||||
|
|
||||||
|
final String missingResource = "org/egothor/stemmer/missing-dictionary.dict.gz";
|
||||||
|
final IOException exception = assertThrows(IOException.class,
|
||||||
|
() -> StemmerPatchTrieLoader.openBundledResource(missingResource));
|
||||||
|
|
||||||
|
assertEquals("Stemmer resource not found: " + missingResource, exception.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Focused filesystem and parser behavior tests.
|
* Focused filesystem and parser behavior tests.
|
||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
@DisplayName("Filesystem and parser behavior")
|
@DisplayName("Filesystem and parser behavior")
|
||||||
|
@Tag("io")
|
||||||
|
@Tag("construction")
|
||||||
|
@Tag("integration")
|
||||||
|
@Tag("trie")
|
||||||
final class FilesystemAndParserTests {
|
final class FilesystemAndParserTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -512,6 +597,44 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that binary load overloads with an explicit dense lookup span
|
||||||
|
* preserve trie semantics while honoring the dense-layout override.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("Binary dense-span override overloads should load equivalent tries")
|
||||||
|
void shouldLoadBinaryWithDenseSpanOverrideOverloads() throws IOException {
|
||||||
|
final Path dictionaryFile = writeDictionary("""
|
||||||
|
run running runs runner
|
||||||
|
city cities
|
||||||
|
study studies studying
|
||||||
|
""");
|
||||||
|
final Path binaryFile = tempDir.resolve("stemmer-trie-overrides.bin.gz");
|
||||||
|
|
||||||
|
final FrequencyTrie<String> original = StemmerPatchTrieLoader.load(dictionaryFile, true,
|
||||||
|
DEFAULT_REDUCTION_MODE);
|
||||||
|
|
||||||
|
StemmerPatchTrieLoader.saveBinary(original, binaryFile);
|
||||||
|
|
||||||
|
final FrequencyTrie<String> fromPathDefault = StemmerPatchTrieLoader.loadBinary(binaryFile);
|
||||||
|
final FrequencyTrie<String> fromPathDefaultByNegative = StemmerPatchTrieLoader.loadBinary(binaryFile,
|
||||||
|
FrequencyTrie.DEFAULT_MAX_EXPANDED_INDEX);
|
||||||
|
final FrequencyTrie<String> fromPathNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile, 0);
|
||||||
|
final FrequencyTrie<String> fromStringNoDense = StemmerPatchTrieLoader.loadBinary(binaryFile.toString(), 0);
|
||||||
|
|
||||||
|
assertTriePatchSemanticsEqual(original, fromPathDefault, "run", "running", "runner", "cities", "studying");
|
||||||
|
assertTriePatchSemanticsEqual(original, fromPathDefaultByNegative, "run", "running", "runner", "cities",
|
||||||
|
"studying");
|
||||||
|
assertTriePatchSemanticsEqual(original, fromPathNoDense, "run", "running", "runner", "cities", "studying");
|
||||||
|
assertTriePatchSemanticsEqual(original, fromStringNoDense, "run", "running", "runner", "cities",
|
||||||
|
"studying");
|
||||||
|
|
||||||
|
assertFalse(fromPathNoDense.root().hasDenseLookup(),
|
||||||
|
"Zero span should disable dense lookup on the loaded root.");
|
||||||
|
assertFalse(fromStringNoDense.root().hasDenseLookup(),
|
||||||
|
"Zero span should disable dense lookup on the loaded root.");
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes a dictionary file into the temporary directory.
|
* Writes a dictionary file into the temporary directory.
|
||||||
*
|
*
|
||||||
@@ -530,7 +653,11 @@ final class StemmerPatchTrieLoaderTest {
|
|||||||
* Bundled dictionary integration tests.
|
* Bundled dictionary integration tests.
|
||||||
*/
|
*/
|
||||||
@Nested
|
@Nested
|
||||||
|
@Tag("slow")
|
||||||
@DisplayName("Bundled dictionaries")
|
@DisplayName("Bundled dictionaries")
|
||||||
|
@Tag("compat")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("regression")
|
||||||
final class BundledDictionaryTests {
|
final class BundledDictionaryTests {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ import java.util.Set;
|
|||||||
import net.jqwik.api.ForAll;
|
import net.jqwik.api.ForAll;
|
||||||
import net.jqwik.api.Label;
|
import net.jqwik.api.Label;
|
||||||
import net.jqwik.api.Property;
|
import net.jqwik.api.Property;
|
||||||
import net.jqwik.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Property-based tests for patch-command stemmer tries.
|
* Property-based tests for patch-command stemmer tries.
|
||||||
@@ -56,9 +56,8 @@ import net.jqwik.api.Tag;
|
|||||||
* persistence must not alter that behavior.
|
* persistence must not alter that behavior.
|
||||||
*/
|
*/
|
||||||
@Label("Stemmer patch trie properties")
|
@Label("Stemmer patch trie properties")
|
||||||
@Tag("unit")
|
|
||||||
@Tag("property")
|
@Tag("property")
|
||||||
@Tag("stemming")
|
@Tag("stemmer")
|
||||||
class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
|
class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
|
|||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
|
@Tag("metadata")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("validation")
|
||||||
@DisplayName("TrieMetadata")
|
@DisplayName("TrieMetadata")
|
||||||
class TrieMetadataTest {
|
class TrieMetadataTest {
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ import org.junit.jupiter.api.Tag;
|
|||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
|
@Tag("core")
|
||||||
|
@Tag("stemmer")
|
||||||
|
@Tag("validation")
|
||||||
@DisplayName("WordTraversalDirection")
|
@DisplayName("WordTraversalDirection")
|
||||||
class WordTraversalDirectionTest {
|
class WordTraversalDirectionTest {
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link ChildDescriptor}.
|
* Unit tests for {@link ChildDescriptor}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("ChildDescriptor")
|
@DisplayName("ChildDescriptor")
|
||||||
class ChildDescriptorTest {
|
class ChildDescriptorTest {
|
||||||
|
|
||||||
|
|||||||
@@ -31,8 +31,10 @@
|
|||||||
package org.egothor.stemmer.trie;
|
package org.egothor.stemmer.trie;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
import org.junit.jupiter.api.DisplayName;
|
import org.junit.jupiter.api.DisplayName;
|
||||||
import org.junit.jupiter.api.Tag;
|
import org.junit.jupiter.api.Tag;
|
||||||
@@ -43,11 +45,40 @@ import org.junit.jupiter.api.Test;
|
|||||||
* documented backing-array exposure.
|
* documented backing-array exposure.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
|
||||||
@Tag("trie")
|
@Tag("trie")
|
||||||
|
@Tag("lookup")
|
||||||
@DisplayName("CompiledNode and NodeData")
|
@DisplayName("CompiledNode and NodeData")
|
||||||
class CompiledNodeAndNodeDataTest {
|
class CompiledNodeAndNodeDataTest {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a typed child array for compiled-node tests.
|
||||||
|
*
|
||||||
|
* @param length requested array length
|
||||||
|
* @return typed child array
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
private static CompiledNode<String>[] children(final int length) {
|
||||||
|
return new CompiledNode[length];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates an empty child array for leaf compiled-node tests.
|
||||||
|
*
|
||||||
|
* @return empty typed child array
|
||||||
|
*/
|
||||||
|
private static CompiledNode<String>[] noChildren() {
|
||||||
|
return children(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a leaf node used as a child in lookup tests.
|
||||||
|
*
|
||||||
|
* @return leaf node
|
||||||
|
*/
|
||||||
|
private static CompiledNode<String> leaf() {
|
||||||
|
return new CompiledNode<>(new char[0], noChildren(), new String[0], new int[0]);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies that {@link NodeData} rejects mismatched edge-related array lengths.
|
* Verifies that {@link NodeData} rejects mismatched edge-related array lengths.
|
||||||
*/
|
*/
|
||||||
@@ -98,8 +129,7 @@ class CompiledNodeAndNodeDataTest {
|
|||||||
@Test
|
@Test
|
||||||
@DisplayName("CompiledNode rejects mismatched edge and child arrays")
|
@DisplayName("CompiledNode rejects mismatched edge and child arrays")
|
||||||
void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() {
|
void compiledNodeShouldRejectMismatchedEdgeAndChildArrays() {
|
||||||
@SuppressWarnings("unchecked")
|
final CompiledNode<String>[] children = noChildren();
|
||||||
final CompiledNode<String>[] children = new CompiledNode[0];
|
|
||||||
|
|
||||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
() -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0]));
|
() -> new CompiledNode<String>(new char[] { 'a' }, children, new String[0], new int[0]));
|
||||||
@@ -113,8 +143,7 @@ class CompiledNodeAndNodeDataTest {
|
|||||||
@Test
|
@Test
|
||||||
@DisplayName("CompiledNode rejects mismatched value arrays")
|
@DisplayName("CompiledNode rejects mismatched value arrays")
|
||||||
void compiledNodeShouldRejectMismatchedValueArrays() {
|
void compiledNodeShouldRejectMismatchedValueArrays() {
|
||||||
@SuppressWarnings("unchecked")
|
final CompiledNode<String>[] children = noChildren();
|
||||||
final CompiledNode<String>[] children = new CompiledNode[0];
|
|
||||||
|
|
||||||
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||||
() -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0]));
|
() -> new CompiledNode<String>(new char[0], children, new String[] { "stem" }, new int[0]));
|
||||||
@@ -130,8 +159,7 @@ class CompiledNodeAndNodeDataTest {
|
|||||||
@DisplayName("CompiledNode accessors expose documented backing arrays")
|
@DisplayName("CompiledNode accessors expose documented backing arrays")
|
||||||
void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() {
|
void compiledNodeAccessorsShouldExposeDocumentedBackingArrays() {
|
||||||
final char[] edgeLabels = new char[] { 'a' };
|
final char[] edgeLabels = new char[] { 'a' };
|
||||||
@SuppressWarnings("unchecked")
|
final CompiledNode<String>[] children = children(1);
|
||||||
final CompiledNode<String>[] children = new CompiledNode[1];
|
|
||||||
final String[] orderedValues = new String[] { "stem" };
|
final String[] orderedValues = new String[] { "stem" };
|
||||||
final int[] orderedCounts = new int[] { 5 };
|
final int[] orderedCounts = new int[] { 5 };
|
||||||
final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts);
|
final CompiledNode<String> node = new CompiledNode<>(edgeLabels, children, orderedValues, orderedCounts);
|
||||||
@@ -141,4 +169,130 @@ class CompiledNodeAndNodeDataTest {
|
|||||||
assertSame(orderedValues, node.orderedValues());
|
assertSame(orderedValues, node.orderedValues());
|
||||||
assertSame(orderedCounts, node.orderedCounts());
|
assertSame(orderedCounts, node.orderedCounts());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that dense lookup is used when the interval is compact.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("CompiledNode can resolve child via dense lookup table")
|
||||||
|
void compiledNodeUsesDenseLookupForCompactIntervals() {
|
||||||
|
final CompiledNode<String>[] children = children(4);
|
||||||
|
children[0] = leaf();
|
||||||
|
children[1] = leaf();
|
||||||
|
children[2] = leaf();
|
||||||
|
children[3] = leaf();
|
||||||
|
|
||||||
|
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'b', 'c', 'd' }, children,
|
||||||
|
new String[] { "1", "2", "3", "4" }, new int[] { 1, 1, 1, 1 });
|
||||||
|
|
||||||
|
assertTrue(node.hasDenseLookup());
|
||||||
|
|
||||||
|
assertSame(children[0], node.findChild('a'));
|
||||||
|
assertSame(children[3], node.findChild('d'));
|
||||||
|
assertSame(null, node.findChild('z'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that fallback linear scan is used for small node degree.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("CompiledNode resolves child by linear scan for small degree")
|
||||||
|
void compiledNodeUsesLinearScanForSmallDegree() {
|
||||||
|
final CompiledNode<String>[] children = children(4);
|
||||||
|
final CompiledNode<String> childA = leaf();
|
||||||
|
final CompiledNode<String> childB = leaf();
|
||||||
|
final CompiledNode<String> childC = leaf();
|
||||||
|
final CompiledNode<String> childD = leaf();
|
||||||
|
children[0] = childA;
|
||||||
|
children[1] = childB;
|
||||||
|
children[2] = childC;
|
||||||
|
children[3] = childD;
|
||||||
|
|
||||||
|
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'z', '中', '你' }, children,
|
||||||
|
new String[] { "1", "2", "3", "4" }, 0, new int[] { 1, 1, 1, 1 });
|
||||||
|
|
||||||
|
assertFalse(node.hasDenseLookup());
|
||||||
|
|
||||||
|
assertSame(childA, node.findChild('a'));
|
||||||
|
assertSame(childD, node.findChild('你'));
|
||||||
|
assertSame(null, node.findChild('b'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that fallback binary search is used for larger node degree without
|
||||||
|
* dense lookup.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("CompiledNode resolves child by binary search for large degree")
|
||||||
|
void compiledNodeUsesBinarySearchForLargeDegree() {
|
||||||
|
final CompiledNode<String>[] children = children(5);
|
||||||
|
final CompiledNode<String> childA = leaf();
|
||||||
|
final CompiledNode<String> childB = leaf();
|
||||||
|
final CompiledNode<String> childC = leaf();
|
||||||
|
final CompiledNode<String> childD = leaf();
|
||||||
|
final CompiledNode<String> childE = leaf();
|
||||||
|
children[0] = childA;
|
||||||
|
children[1] = childB;
|
||||||
|
children[2] = childC;
|
||||||
|
children[3] = childD;
|
||||||
|
children[4] = childE;
|
||||||
|
|
||||||
|
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a', 'c', 'k', 't', 'z' }, children,
|
||||||
|
new String[] { "1", "2", "3", "4", "5" }, 0, new int[] { 1, 1, 1, 1, 1 });
|
||||||
|
|
||||||
|
assertFalse(node.hasDenseLookup());
|
||||||
|
|
||||||
|
assertSame(childC, node.findChild('k'));
|
||||||
|
assertSame(childE, node.findChild('z'));
|
||||||
|
assertSame(null, node.findChild('x'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies the basic node-state helpers that are used by diagnostics and
|
||||||
|
* behavioral checks.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("CompiledNode reports leaf, value and edge presence state")
|
||||||
|
void compiledNodeReportsNodeStateHelpers() {
|
||||||
|
final CompiledNode<String>[] childless = noChildren();
|
||||||
|
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], childless, new String[0], new int[0]);
|
||||||
|
|
||||||
|
assertTrue(leaf.isLeaf());
|
||||||
|
assertFalse(leaf.hasChildren());
|
||||||
|
assertFalse(leaf.hasValues());
|
||||||
|
assertFalse(leaf.hasEdge('a'));
|
||||||
|
|
||||||
|
final CompiledNode<String>[] child = children(1);
|
||||||
|
final String[] orderedValues = new String[] { "leaf" };
|
||||||
|
final int[] orderedCounts = new int[] { 1 };
|
||||||
|
child[0] = new CompiledNode<>(new char[0], noChildren(), orderedValues, orderedCounts);
|
||||||
|
final CompiledNode<String> node = new CompiledNode<>(new char[] { 'a' }, child, orderedValues, orderedCounts);
|
||||||
|
|
||||||
|
assertFalse(node.isLeaf());
|
||||||
|
assertTrue(node.hasChildren());
|
||||||
|
assertTrue(node.hasValues());
|
||||||
|
assertTrue(node.valueCount() > 0);
|
||||||
|
assertTrue(node.hasEdge('a'));
|
||||||
|
assertFalse(node.hasEdge('b'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies structural equality and hash-code behavior for compiled nodes.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
@DisplayName("CompiledNode equals and hashCode align for identical structure")
|
||||||
|
void compiledNodeEqualsAndHashCodeAlignForIdenticalStructure() {
|
||||||
|
final CompiledNode<String>[] child = children(1);
|
||||||
|
final CompiledNode<String> leaf = new CompiledNode<>(new char[0], noChildren(), new String[] { "v" },
|
||||||
|
new int[] { 1 });
|
||||||
|
child[0] = leaf;
|
||||||
|
|
||||||
|
final CompiledNode<String> first = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
|
||||||
|
new int[] { 2 });
|
||||||
|
final CompiledNode<String> second = new CompiledNode<>(new char[] { 'a' }, child, new String[] { "x" },
|
||||||
|
new int[] { 2 });
|
||||||
|
|
||||||
|
assertEquals(first, second);
|
||||||
|
assertEquals(first.hashCode(), second.hashCode());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link DominantLocalDescriptor}.
|
* Unit tests for {@link DominantLocalDescriptor}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("DominantLocalDescriptor")
|
@DisplayName("DominantLocalDescriptor")
|
||||||
class DominantLocalDescriptorTest {
|
class DominantLocalDescriptorTest {
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link LocalValueSummary}.
|
* Unit tests for {@link LocalValueSummary}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("LocalValueSummary")
|
@DisplayName("LocalValueSummary")
|
||||||
class LocalValueSummaryTest {
|
class LocalValueSummaryTest {
|
||||||
|
|
||||||
@@ -190,6 +191,9 @@ class LocalValueSummaryTest {
|
|||||||
/**
|
/**
|
||||||
* Test helper with identical textual form but distinct identity.
|
* Test helper with identical textual form but distinct identity.
|
||||||
*/
|
*/
|
||||||
|
@Tag("unit")
|
||||||
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
private static final class TextTwin {
|
private static final class TextTwin {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -44,7 +44,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link MutableNode}.
|
* Unit tests for {@link MutableNode}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("construction")
|
||||||
@DisplayName("MutableNode")
|
@DisplayName("MutableNode")
|
||||||
class MutableNodeTest {
|
class MutableNodeTest {
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link RankedLocalDescriptor}.
|
* Unit tests for {@link RankedLocalDescriptor}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("RankedLocalDescriptor")
|
@DisplayName("RankedLocalDescriptor")
|
||||||
class RankedLocalDescriptorTest {
|
class RankedLocalDescriptorTest {
|
||||||
|
|
||||||
|
|||||||
@@ -48,7 +48,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link ReducedNode}.
|
* Unit tests for {@link ReducedNode}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("ReducedNode")
|
@DisplayName("ReducedNode")
|
||||||
class ReducedNodeTest {
|
class ReducedNodeTest {
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link ReductionContext}.
|
* Unit tests for {@link ReductionContext}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("ReductionContext")
|
@DisplayName("ReductionContext")
|
||||||
class ReductionContextTest {
|
class ReductionContextTest {
|
||||||
|
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link ReductionSignature}.
|
* Unit tests for {@link ReductionSignature}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("ReductionSignature")
|
@DisplayName("ReductionSignature")
|
||||||
class ReductionSignatureTest {
|
class ReductionSignatureTest {
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,8 @@ import org.junit.jupiter.api.Test;
|
|||||||
* Unit tests for {@link UnorderedLocalDescriptor}.
|
* Unit tests for {@link UnorderedLocalDescriptor}.
|
||||||
*/
|
*/
|
||||||
@Tag("unit")
|
@Tag("unit")
|
||||||
@Tag("fast")
|
@Tag("trie")
|
||||||
|
@Tag("reduction")
|
||||||
@DisplayName("UnorderedLocalDescriptor")
|
@DisplayName("UnorderedLocalDescriptor")
|
||||||
class UnorderedLocalDescriptorTest {
|
class UnorderedLocalDescriptorTest {
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user