Compare commits
14 Commits
release@1.
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
6ccce248ea
|
|||
|
5a511374f3
|
|||
|
48f21cab72
|
|||
|
39969463a2
|
|||
|
6dbdb4bae8
|
|||
|
2ab3e74048
|
|||
|
128fa919f2
|
|||
|
1f5decd6ea
|
|||
|
9eee321fef
|
|||
|
3e0f786042
|
|||
|
041b7f43fb
|
|||
|
8785f2b7cb
|
|||
|
4d939f5b6e
|
|||
|
a9d15fa3ae
|
46
.classpath
46
.classpath
@@ -1,46 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="bin/main" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="main"/>
|
||||
<attribute name="gradle_used_by_scope" value="main,test,jmh"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/test" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="test"/>
|
||||
<attribute name="gradle_used_by_scope" value="test,jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/main" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="main"/>
|
||||
<attribute name="gradle_used_by_scope" value="main,test,jmh"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/jmh" path="src/jmh/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="jmh"/>
|
||||
<attribute name="gradle_used_by_scope" value="jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/jmh" path="build/third-party/snowball/source/libstemmer_java-3.0.1/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="jmh"/>
|
||||
<attribute name="gradle_used_by_scope" value="jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/test" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="test"/>
|
||||
<attribute name="gradle_used_by_scope" value="test,jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-21/"/>
|
||||
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
|
||||
<classpathentry kind="output" path="bin/default"/>
|
||||
</classpath>
|
||||
4
.ruleset
4
.ruleset
@@ -162,12 +162,12 @@
|
||||
<rule ref="category/java/design.xml/CollapsibleIfStatements"/>
|
||||
<rule ref="category/java/design.xml/CouplingBetweenObjects">
|
||||
<properties>
|
||||
<property name="threshold" value="50" />
|
||||
<property name="threshold" value="60" />
|
||||
</properties>
|
||||
</rule>
|
||||
<rule ref="category/java/design.xml/CyclomaticComplexity">
|
||||
<properties>
|
||||
<property name="methodReportLevel" value="18" />
|
||||
<property name="methodReportLevel" value="19" />
|
||||
</properties>
|
||||
</rule>
|
||||
<rule ref="category/java/design.xml/DataClass"/>
|
||||
|
||||
29
LICENSE-stemmer-data
Normal file
29
LICENSE-stemmer-data
Normal file
@@ -0,0 +1,29 @@
|
||||
Stemmer data licensing
|
||||
|
||||
The software source code in this repository is licensed separately under
|
||||
the BSD 3-Clause License.
|
||||
|
||||
Stemmer dictionary and morphology data files are not covered by
|
||||
the BSD 3-Clause License unless explicitly stated otherwise.
|
||||
|
||||
This repository contains adapted data derived from the UniMorph project:
|
||||
https://unimorph.github.io/
|
||||
|
||||
Only stemmer data derived from sources that permit commercial use are included
|
||||
in the main distribution of this repository.
|
||||
|
||||
Accepted upstream licenses for distributed stemmer data in this repository:
|
||||
- CC BY-SA 3.0
|
||||
- CC BY-SA 4.0
|
||||
- CC BY 4.0
|
||||
|
||||
Sources under non-commercial licenses, including CC BY-NC-SA 4.0, are excluded
|
||||
from the main distribution.
|
||||
|
||||
Modifications in this repository may include cleaning, normalization,
|
||||
deduplication, filtering, conversion, and reformatting.
|
||||
|
||||
Copyright (c) 2026 Leo Galambos for the modifications, to the extent permitted
|
||||
by the applicable upstream license terms.
|
||||
|
||||
Per-file licensing is stated in the header of each generated stemmer data file.
|
||||
@@ -54,7 +54,7 @@ Radixor is especially attractive when you want something more adaptable than sim
|
||||
|
||||
Radixor includes a JMH benchmark suite for both its own algorithmic core and a side-by-side English comparison against the Snowball Porter stemmer family.
|
||||
|
||||
On the current English comparison workload, Radixor with bundled `US_UK_PROFI` reaches approximately **31 to 32 million tokens per second**. Snowball original Porter reaches approximately **8 million tokens per second**, and Snowball English (Porter2) approximately **5 to 5.5 million tokens per second**.
|
||||
On the current English comparison workload, Radixor with bundled `US_UK` reaches approximately **31 to 32 million tokens per second**. Snowball original Porter reaches approximately **8 million tokens per second**, and Snowball English (Porter2) approximately **5 to 5.5 million tokens per second**.
|
||||
|
||||
That places Radixor at approximately:
|
||||
|
||||
@@ -137,7 +137,7 @@ The repository keeps the front page concise and places detailed documentation un
|
||||
A practical first guide to loading, compiling, and using Radixor.
|
||||
|
||||
- [Built-in Languages](docs/built-in-languages.md)
|
||||
Overview of bundled language resources such as `US_UK` and `US_UK_PROFI`.
|
||||
Overview of bundled language resources such as `US_UK`.
|
||||
|
||||
- [Dictionary Format](docs/dictionary-format.md)
|
||||
How to write and normalize stemming dictionaries.
|
||||
|
||||
38
build.gradle
38
build.gradle
@@ -116,6 +116,14 @@ tasks.withType(Test).configureEach {
|
||||
jvmArgs "-javaagent:${configurations.mockitoAgent.singleFile}"
|
||||
}
|
||||
|
||||
/*
|
||||
* Bundled dictionary integration tests compile and reload large real-world
|
||||
* stemming dictionaries, including large language resources such as es_es.
|
||||
* The default Gradle test executor heap is too small for this workload.
|
||||
*/
|
||||
minHeapSize = '1g'
|
||||
maxHeapSize = '4g'
|
||||
|
||||
finalizedBy(tasks.named('jacocoTestReport'))
|
||||
|
||||
reports {
|
||||
@@ -134,6 +142,13 @@ tasks.withType(Pmd).configureEach {
|
||||
tasks.named('jacocoTestReport', JacocoReport) {
|
||||
dependsOn(tasks.named('test'))
|
||||
|
||||
classDirectories.setFrom(
|
||||
files(sourceSets.main.output).asFileTree.matching {
|
||||
exclude 'org/egothor/stemmer/StemmerKnowledgeExperiment*'
|
||||
exclude 'org/egothor/stemmer/DiacriticStripper*'
|
||||
}
|
||||
)
|
||||
|
||||
reports {
|
||||
xml.required = true
|
||||
csv.required = false
|
||||
@@ -178,7 +193,17 @@ pitest {
|
||||
'org.egothor.stemmer.trie.*Test'
|
||||
]
|
||||
|
||||
excludedClasses = ['org.egothor.stemmer.Compile']
|
||||
excludedClasses = [
|
||||
'org.egothor.stemmer.Compile*',
|
||||
'org.egothor.stemmer.StemmerPatchTrieLoader*',
|
||||
'org.egothor.stemmer.StemmerKnowledgeExperiment*',
|
||||
'org.egothor.stemmer.StemmerKnowledgeExperimentCli*'
|
||||
]
|
||||
excludedTestClasses = [
|
||||
'org.egothor.stemmer.CompileIntegrationTest',
|
||||
'org.egothor.stemmer.StemmerPatchTrieLoaderTest',
|
||||
'org.egothor.stemmer.StemmerKnowledgeExperimentTest'
|
||||
]
|
||||
outputFormats = ['XML', 'HTML']
|
||||
timestampedReports = false
|
||||
exportLineCoverage = true
|
||||
@@ -192,6 +217,13 @@ application {
|
||||
executableDir = 'bin'
|
||||
}
|
||||
|
||||
tasks.register('stemmerKnowledgeExperiment', JavaExec) {
|
||||
group = 'application'
|
||||
description = 'Runs the stemmer knowledge evaluation experiment.'
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
mainClass = 'org.egothor.stemmer.StemmerKnowledgeExperimentCli'
|
||||
}
|
||||
|
||||
distributions {
|
||||
main {
|
||||
distributionBaseName = 'radixor'
|
||||
@@ -205,6 +237,10 @@ distributions {
|
||||
into ''
|
||||
}
|
||||
|
||||
from('LICENSE-stemmer-data') {
|
||||
into ''
|
||||
}
|
||||
|
||||
from('docs') {
|
||||
into 'docs'
|
||||
include '**/*.md'
|
||||
|
||||
@@ -25,11 +25,11 @@ Each stage has a different purpose.
|
||||
The textual dictionary groups known word forms under a canonical stem:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
```
|
||||
|
||||
The first token is the canonical stem. The following tokens are known variants.
|
||||
The first column is the canonical stem. The following tab-separated columns are known variants.
|
||||
|
||||
### Patch-command generation
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ The benchmark suite currently covers two categories:
|
||||
|
||||
The comparison benchmark processes the same deterministic English token stream through:
|
||||
|
||||
- Radixor with bundled `US_UK_PROFI`,
|
||||
- Radixor with bundled `US_UK` (older benchmark snapshots used the now-retired `US_UK_PROFI` resource),
|
||||
- Snowball original Porter,
|
||||
- Snowball English, commonly referred to as Porter2.
|
||||
|
||||
@@ -37,7 +37,7 @@ For that reason, the published badge values should be treated primarily as a com
|
||||
|
||||
A recent JMH run on JDK 21.0.10 with JMH 1.37, one thread, three warmup iterations, and five measurement iterations produced the following approximate throughput ranges:
|
||||
|
||||
| Workload | Radixor `US_UK_PROFI` | Snowball Porter | Snowball English |
|
||||
| Workload | Radixor `US_UK` *(historical runs: `US_UK_PROFI`)* | Snowball Porter | Snowball English |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| About 12,000 generated tokens | 30.99 M tokens/s | 8.21 M tokens/s | 5.46 M tokens/s |
|
||||
| About 60,000 generated tokens | 32.25 M tokens/s | 8.02 M tokens/s | 5.11 M tokens/s |
|
||||
@@ -83,7 +83,7 @@ The workload intentionally mixes:
|
||||
- simple inflections,
|
||||
- common derivational forms,
|
||||
- US and UK spelling families,
|
||||
- lexical forms appropriate for `US_UK_PROFI`.
|
||||
- lexical forms appropriate for the current bundled `US_UK` resource (with historical continuity from earlier `US_UK_PROFI` runs).
|
||||
|
||||
This design keeps runs reproducible across environments and avoids accidental drift caused by changing external corpora.
|
||||
|
||||
|
||||
@@ -1,41 +1,49 @@
|
||||
# Built-in Languages
|
||||
|
||||
Radixor provides a set of bundled stemmer dictionaries that can be loaded directly without preparing custom lexical data first.
|
||||
|
||||
These resources are intended as practical default dictionaries for common use. They provide a solid starting point for evaluation, integration, and general-purpose stemming workloads, while still fitting naturally into workflows where the bundled baseline is later refined, extended, or replaced by a custom dictionary.
|
||||
Radixor ships with a curated set of bundled stemmer dictionaries that can be loaded directly from the library distribution. These resources are intended to provide an immediately usable baseline for evaluation, prototyping, integration, and general-purpose stemming workloads, while still fitting naturally into workflows where the bundled baseline is later refined, extended, or replaced with custom lexical data.
|
||||
|
||||
## Overview
|
||||
|
||||
Bundled dictionaries are exposed through:
|
||||
|
||||
```java
|
||||
StemmerPatchTrieLoader.Language
|
||||
org.egothor.stemmer.StemmerPatchTrieLoader.Language
|
||||
```
|
||||
|
||||
They are packaged with the library as text resources and compiled into a `FrequencyTrie<String>` when loaded.
|
||||
Each bundled dictionary is packaged with the library as a compressed UTF-8 text resource. When loaded, the resource is parsed by `StemmerDictionaryParser`, transformed into patch-command mappings, and compiled into a read-only `FrequencyTrie<String>` by `StemmerPatchTrieLoader`.
|
||||
|
||||
## Supported languages
|
||||
The bundled language definition also carries a language-level right-to-left flag. That flag is used by the loader to derive the `WordTraversalDirection` used for both trie-key construction and patch-command generation. In practice, left-to-right bundled languages use historical backward Egothor traversal, while right-to-left bundled languages use forward traversal over the stored form.
|
||||
|
||||
## Supported bundled languages
|
||||
|
||||
The following bundled language identifiers are currently available:
|
||||
|
||||
| Language | Enum constant | Notes |
|
||||
|---|---|---|
|
||||
| Danish | `DA_DK` | Bundled general-purpose dictionary |
|
||||
| German | `DE_DE` | Bundled general-purpose dictionary |
|
||||
| Spanish | `ES_ES` | Bundled general-purpose dictionary |
|
||||
| French | `FR_FR` | Bundled general-purpose dictionary |
|
||||
| Italian | `IT_IT` | Bundled general-purpose dictionary |
|
||||
| Dutch | `NL_NL` | Bundled general-purpose dictionary |
|
||||
| Norwegian | `NO_NO` | Bundled general-purpose dictionary |
|
||||
| Portuguese | `PT_PT` | Bundled general-purpose dictionary |
|
||||
| Russian | `RU_RU` | Currently supplied in normalized transliterated form |
|
||||
| Swedish | `SV_SE` | Bundled general-purpose dictionary |
|
||||
| English | `US_UK` | Standard English dictionary |
|
||||
| English | `US_UK_PROFI` | Extended English dictionary |
|
||||
| Language | Enum constant | Writing direction | Notes |
|
||||
|---|---|---:|---|
|
||||
| Czech | `CS_CZ` | LTR | Bundled general-purpose dictionary |
|
||||
| Danish | `DA_DK` | LTR | Bundled general-purpose dictionary |
|
||||
| German | `DE_DE` | LTR | Bundled general-purpose dictionary |
|
||||
| Spanish | `ES_ES` | LTR | Bundled general-purpose dictionary |
|
||||
| Persian | `FA_IR` | RTL | Bundled dictionary uses forward traversal over the stored form |
|
||||
| Finnish | `FI_FI` | LTR | Bundled general-purpose dictionary |
|
||||
| French | `FR_FR` | LTR | Bundled general-purpose dictionary |
|
||||
| Hebrew | `HE_IL` | RTL | Bundled dictionary uses forward traversal over the stored form |
|
||||
| Hungarian | `HU_HU` | LTR | Bundled general-purpose dictionary |
|
||||
| Italian | `IT_IT` | LTR | Bundled general-purpose dictionary |
|
||||
| Norwegian Bokmål | `NB_NO` | LTR | Bundled general-purpose dictionary |
|
||||
| Dutch | `NL_NL` | LTR | Bundled general-purpose dictionary |
|
||||
| Norwegian Nynorsk | `NN_NO` | LTR | Bundled general-purpose dictionary |
|
||||
| Polish | `PL_PL` | LTR | Bundled general-purpose dictionary |
|
||||
| Portuguese | `PT_PT` | LTR | Bundled general-purpose dictionary |
|
||||
| Russian | `RU_RU` | LTR | Bundled general-purpose dictionary |
|
||||
| Swedish | `SV_SE` | LTR | Bundled general-purpose dictionary |
|
||||
| Ukrainian | `UK_UA` | LTR | Bundled general-purpose dictionary |
|
||||
| English | `US_UK` | LTR | Bundled general-purpose dictionary |
|
||||
| Yiddish | `YI` | RTL | Bundled dictionary uses forward traversal over the stored form |
|
||||
|
||||
## Basic usage
|
||||
|
||||
Load a bundled stemmer like this:
|
||||
Load a bundled dictionary like this:
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
@@ -52,16 +60,18 @@ public final class BuiltInExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
System.out.println(trie.traversalDirection());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The loader reads the bundled dictionary resource, parses the textual entries, derives patch-command mappings, and compiles the result into a read-only trie.
|
||||
This call loads the bundled dictionary resource for the selected language, parses its lexical entries, derives patch-command mappings, and compiles the result into a read-only trie.
|
||||
|
||||
## Example: stemming with `US_UK_PROFI`
|
||||
## Example: stemming with a bundled dictionary
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
@@ -79,44 +89,49 @@ public final class EnglishExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
final String word = "running";
|
||||
final String patch = trie.get(word);
|
||||
final String stem = PatchCommandEncoder.apply(word, patch);
|
||||
final String stem = PatchCommandEncoder.apply(word, patch, trie.traversalDirection());
|
||||
|
||||
System.out.println(word + " -> " + stem);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## `US_UK` and `US_UK_PROFI`
|
||||
Passing `trie.traversalDirection()` to `PatchCommandEncoder.apply(...)` is the correct general contract. It ensures that the patch is applied using the same logical traversal model that was used when the trie and its patch commands were produced.
|
||||
|
||||
Radixor currently provides two bundled English variants.
|
||||
## Traversal behavior and right-to-left languages
|
||||
|
||||
### `US_UK`
|
||||
Bundled dictionaries are not all processed identically.
|
||||
|
||||
`US_UK` is the lighter-weight bundled English resource. It is suitable where a smaller default dictionary is preferred and maximal lexical coverage is not the primary goal.
|
||||
For traditional left-to-right suffix-oriented resources, Radixor preserves historical Egothor behavior and traverses logical word characters backward. That means trie paths are constructed from the logical end of the stored word toward its beginning, and patch commands are interpreted with the same backward traversal model.
|
||||
|
||||
### `US_UK_PROFI`
|
||||
For bundled right-to-left languages such as Persian, Hebrew, and Yiddish, Radixor uses forward traversal over the stored form. In those cases:
|
||||
|
||||
`US_UK_PROFI` is the more extensive bundled English resource. It offers broader lexical coverage and is the better default for most applications that want stronger out-of-the-box behavior.
|
||||
- trie keys are traversed from the logical beginning of the stored form,
|
||||
- patch commands are generated in that same forward direction,
|
||||
- patch application must use `WordTraversalDirection.FORWARD`, which is naturally obtained from `trie.traversalDirection()`.
|
||||
|
||||
### Recommendation
|
||||
This design keeps the traversal policy explicit and consistent across dictionary loading, trie lookup, binary persistence, builder reconstruction, and patch application.
|
||||
|
||||
For most English-language deployments, prefer:
|
||||
## Reduction behavior
|
||||
|
||||
```text
|
||||
US_UK_PROFI
|
||||
```
|
||||
Bundled dictionaries can be compiled using any supported `ReductionMode`. The reduction configuration controls how semantically equivalent subtrees are merged during trie compilation, while preserving the contract of the selected mode.
|
||||
|
||||
Use `US_UK` when a smaller bundled baseline is more appropriate.
|
||||
Typical entry points are:
|
||||
|
||||
- `StemmerPatchTrieLoader.load(language, storeOriginal, reductionMode)`
|
||||
- `StemmerPatchTrieLoader.load(language, storeOriginal, reductionSettings)`
|
||||
|
||||
For most users, `ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS` is the most conservative general-purpose choice because it preserves ranked `getAll(...)` behavior.
|
||||
|
||||
## Intended role of bundled dictionaries
|
||||
|
||||
Bundled dictionaries should be understood as **general-purpose default resources**.
|
||||
Bundled dictionaries should be understood as practical default resources.
|
||||
|
||||
They are a good fit when:
|
||||
|
||||
@@ -125,15 +140,18 @@ They are a good fit when:
|
||||
- a reasonable baseline is sufficient,
|
||||
- the goal is evaluation, prototyping, or straightforward integration.
|
||||
|
||||
They are also well suited to staged refinement workflows in which the bundled base is loaded first, then extended with domain-specific vocabulary, and finally persisted as a custom binary artifact.
|
||||
They are also well suited to staged refinement workflows in which a bundled base is loaded first, then extended with domain-specific vocabulary, and finally persisted as a custom binary artifact.
|
||||
|
||||
## Character representation
|
||||
|
||||
The current bundled resources follow a pragmatic normalization convention.
|
||||
Bundled dictionaries are ordinary UTF-8 lexical resources. The parser reads them as text, the trie stores standard Java strings, and the patch-command model operates on general character sequences.
|
||||
|
||||
At present, bundled dictionaries are supplied in normalized plain-ASCII form. For some languages, this is simply a lightweight maintenance convention. For others, especially languages commonly written in another script, it reflects a transliterated lexical resource. Russian is the clearest example in the current bundled set.
|
||||
This is important for two reasons:
|
||||
|
||||
This convention belongs to the supplied dictionary resources, not to the core stemming model. The parser reads UTF-8 text, the dictionary model works with ordinary Java strings, and the trie and patch-command mechanism operate on general character sequences. In practical terms, the architecture is compatible with native-script dictionaries when suitable lexical resources are available.
|
||||
1. the built-in resources are not limited to ASCII-only processing,
|
||||
2. the traversal model is orthogonal to character encoding and script choice.
|
||||
|
||||
In other words, right-to-left handling in the loader is about logical traversal strategy, not about introducing a separate character model.
|
||||
|
||||
## When to prefer custom dictionaries
|
||||
|
||||
@@ -141,8 +159,8 @@ A custom dictionary is usually the better choice when:
|
||||
|
||||
- domain-specific vocabulary materially affects stemming quality,
|
||||
- lexical coverage must be controlled more precisely,
|
||||
- a stronger language resource is available than the bundled baseline,
|
||||
- native-script support is needed beyond the currently bundled resources.
|
||||
- a stronger lexical resource is available than the bundled baseline,
|
||||
- operational requirements demand an explicitly curated, versioned artifact.
|
||||
|
||||
Typical examples include:
|
||||
|
||||
@@ -150,7 +168,7 @@ Typical examples include:
|
||||
- biomedical language,
|
||||
- legal or financial vocabulary,
|
||||
- organization-specific product and process names,
|
||||
- language resources maintained in native scripts.
|
||||
- dictionaries maintained with project-specific validation rules.
|
||||
|
||||
## Production recommendation
|
||||
|
||||
@@ -158,11 +176,11 @@ For production systems, the most robust workflow is usually:
|
||||
|
||||
1. start from a bundled dictionary when it is suitable,
|
||||
2. extend it with domain-specific forms if needed,
|
||||
3. compile or rebuild it into a binary `.radixor.gz` artifact,
|
||||
4. deploy that compiled artifact,
|
||||
5. load it at runtime using `loadBinary(...)`.
|
||||
3. rebuild it into a binary artifact,
|
||||
4. deploy that compiled binary artifact,
|
||||
5. load it at runtime through `loadBinary(...)`.
|
||||
|
||||
This avoids repeated startup parsing and makes the deployed stemming behavior explicit and versionable.
|
||||
This avoids repeated startup parsing and makes the deployed stemming behavior explicit, reproducible, and versionable.
|
||||
|
||||
## Example refinement workflow
|
||||
|
||||
@@ -185,7 +203,7 @@ public final class BundledRefinementExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> base = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
@@ -204,11 +222,27 @@ public final class BundledRefinementExample {
|
||||
}
|
||||
```
|
||||
|
||||
The reconstructed builder preserves the traversal direction of the source trie, so refinements remain semantically aligned with the original bundled dictionary.
|
||||
|
||||
## Extending language support
|
||||
|
||||
The built-in set is intentionally a practical baseline rather than a closed catalog. High-quality dictionaries for additional languages, improved language coverage, and stronger native-script resources are all natural extension paths for the project.
|
||||
The built-in set is intentionally a practical baseline rather than a closed catalog. Additional languages, stronger lexical coverage, and improved dictionaries for currently supported languages are all natural extension paths.
|
||||
|
||||
What matters most is not only the number of entries, but the quality, consistency, and operational usefulness of the lexical resource being added.
|
||||
What matters most is not only the number of entries, but the quality, consistency, maintainability, and operational usefulness of the lexical resource being added.
|
||||
|
||||
## Related API surface
|
||||
|
||||
The following types are typically involved when working with bundled dictionaries:
|
||||
|
||||
- `StemmerPatchTrieLoader`
|
||||
- `StemmerPatchTrieLoader.Language`
|
||||
- `FrequencyTrie`
|
||||
- `PatchCommandEncoder`
|
||||
- `WordTraversalDirection`
|
||||
- `ReductionMode`
|
||||
- `ReductionSettings`
|
||||
- `StemmerPatchTrieBinaryIO`
|
||||
- `FrequencyTrieBuilders`
|
||||
|
||||
## Next steps
|
||||
|
||||
@@ -219,4 +253,4 @@ What matters most is not only the number of entries, but the quality, consistenc
|
||||
|
||||
## Summary
|
||||
|
||||
Radixor’s built-in language support provides immediate usability, practical default dictionaries, and a strong starting point for custom refinement. The current bundled resources follow a pragmatic normalization convention, while the underlying architecture remains well suited to richer language resources and future extensions.
|
||||
Radixor’s built-in language support provides immediate usability, a professionally defined baseline API, and a practical starting point for custom refinement. The bundled set now includes both left-to-right and right-to-left languages, and the library models that distinction explicitly through `WordTraversalDirection` so that trie construction, lookup, and patch application remain consistent.
|
||||
|
||||
@@ -8,8 +8,8 @@ This is the preferred preparation workflow when stemming should run against an a
|
||||
|
||||
The `Compile` tool performs the following steps:
|
||||
|
||||
1. reads the input dictionary in the standard Radixor stemmer format,
|
||||
2. parses each line into a canonical stem and its known variants,
|
||||
1. reads the input dictionary in the standard Radixor stemmer format, accepting either plain UTF-8 text or GZip-compressed UTF-8 text,
|
||||
2. parses each line into a canonical stem column and its known variant columns,
|
||||
3. converts variants into patch commands,
|
||||
4. builds a mutable trie of patch-command values,
|
||||
5. applies the configured reduction mode,
|
||||
@@ -21,9 +21,10 @@ This workflow is intentionally aligned with the same dictionary semantics used e
|
||||
|
||||
```bash
|
||||
java org.egothor.stemmer.Compile \
|
||||
--input ./data/stemmer.txt \
|
||||
--input ./data/stemmer.tsv \
|
||||
--output ./build/english.radixor.gz \
|
||||
--reduction-mode MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS \
|
||||
--case-processing-mode LOWERCASE_WITH_LOCALE_ROOT \
|
||||
--store-original \
|
||||
--overwrite
|
||||
```
|
||||
@@ -37,6 +38,8 @@ The CLI supports the following arguments:
|
||||
--output <file>
|
||||
--reduction-mode <mode>
|
||||
[--store-original]
|
||||
[--right-to-left]
|
||||
[--case-processing-mode <mode>]
|
||||
[--dominant-winner-min-percent <1..100>]
|
||||
[--dominant-winner-over-second-ratio <1..n>]
|
||||
[--overwrite]
|
||||
@@ -47,12 +50,12 @@ The CLI supports the following arguments:
|
||||
|
||||
Path to the source dictionary file.
|
||||
|
||||
The file must use the standard line-oriented dictionary format. Each non-empty logical line starts with the canonical stem and may contain zero or more variants. The parser expects UTF-8 input, lowercases it using `Locale.ROOT`, and ignores trailing remarks introduced by `#` or `//`.
|
||||
The file must use the standard line-oriented tab-separated values dictionary format, meaning that columns are separated by the tab character. Each non-empty logical line starts with the canonical stem column and may contain zero or more variant columns. The input may be plain UTF-8 text or GZip-compressed UTF-8 text; compression is detected from the stream header rather than the file extension. The parser processes case according to `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores trailing remarks introduced by `#` or `//`, and currently ignores dictionary items containing embedded whitespace while reporting them through warning-level log entries.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
--input ./data/stemmer.txt
|
||||
--input ./data/stemmer.tsv
|
||||
```
|
||||
|
||||
### `--output <file>`
|
||||
@@ -95,6 +98,31 @@ When this flag is present, the canonical stem itself is inserted using the no-op
|
||||
|
||||
This is usually a sensible default for real dictionaries because it ensures that canonical forms are directly representable in the compiled trie rather than relying only on their variants.
|
||||
|
||||
### `--right-to-left`
|
||||
|
||||
When present, compilation uses forward traversal (`WordTraversalDirection.FORWARD`) so stored forms are processed from their logical beginning.
|
||||
|
||||
```text
|
||||
--right-to-left
|
||||
```
|
||||
|
||||
This option is intended for right-to-left languages where affix behavior should operate on the written form without externally reversing words.
|
||||
|
||||
### `--case-processing-mode <mode>`
|
||||
|
||||
Controls dictionary key normalization during compilation and lookup. The setting is stored in persisted trie metadata and is therefore available to runtime lookup after binary loading.
|
||||
|
||||
Supported values are:
|
||||
|
||||
- `LOWERCASE_WITH_LOCALE_ROOT` (default)
|
||||
- `AS_IS`
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
--case-processing-mode AS_IS
|
||||
```
|
||||
|
||||
### `--dominant-winner-min-percent <1..100>`
|
||||
|
||||
Sets the minimum winner percentage used by dominant-result reduction settings.
|
||||
@@ -177,6 +205,8 @@ The CLI is best used as a preparation step during packaging, deployment, or cont
|
||||
|
||||
A `.radixor.gz` file should be handled as a versioned output artifact. It represents a specific dictionary state, a specific reduction mode, and, where relevant, specific dominant-result thresholds.
|
||||
|
||||
Compiled tries also persist a human-readable metadata block (`key=value` lines) that includes format version, traversal direction, RTL indicator, reduction mode, dominant thresholds, diacritic-processing mode, and case-processing mode. After decompression, you can inspect this block directly to identify what dictionary/trie configuration the artifact contains. The current CLI uses `DiacriticProcessingMode.AS_IS`; custom diacritic stripping is available through the programmatic builder and loader APIs rather than through a CLI flag.
|
||||
|
||||
### Choose reduction mode deliberately
|
||||
|
||||
The ranked `getAll()` mode is the safest default. The unordered and dominant modes should be chosen only when their trade-offs are acceptable for the consuming application.
|
||||
@@ -190,15 +220,15 @@ Compilation is usually a one-time step and is generally fast. The more important
|
||||
### 1. Prepare a dictionary
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
connect connected connecting
|
||||
run running runs ran
|
||||
connect connected connecting
|
||||
```
|
||||
|
||||
### 2. Compile it
|
||||
|
||||
```bash
|
||||
java org.egothor.stemmer.Compile \
|
||||
--input ./data/stemmer.txt \
|
||||
--input ./data/stemmer.tsv \
|
||||
--output ./build/english.radixor.gz \
|
||||
--reduction-mode MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS \
|
||||
--store-original
|
||||
|
||||
@@ -22,27 +22,29 @@ In practice, dictionary quality matters more than dictionary size. A smaller but
|
||||
|
||||
## Preferred dictionary shape
|
||||
|
||||
Radixor uses a simple line-oriented format:
|
||||
Radixor uses a simple line-oriented tab-separated values format, meaning that columns are separated by the tab character:
|
||||
|
||||
```text
|
||||
<stem> <variant1> <variant2> <variant3> ...
|
||||
<stem> <variant1> <variant2> <variant3> ...
|
||||
```
|
||||
|
||||
The first token on a line is the canonical stem. All following tokens on that line are known variants that should reduce to that stem.
|
||||
The first column on a line is the canonical stem. All following tab-separated columns on that line are known variants that should reduce to that stem.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
```
|
||||
|
||||
The parser:
|
||||
|
||||
- reads UTF-8 text,
|
||||
- normalizes input to lower case using `Locale.ROOT`,
|
||||
- interprets each line as tab-separated values,
|
||||
- applies configurable case processing through `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`),
|
||||
- ignores empty lines,
|
||||
- supports remarks introduced by `#` or `//`.
|
||||
- supports remarks introduced by `#` or `//`,
|
||||
- currently ignores dictionary items containing embedded whitespace and reports them through warning-level log entries.
|
||||
|
||||
For full format details, see [Dictionary format](dictionary-format.md).
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Dictionary Format
|
||||
|
||||
Radixor uses a simple line-oriented dictionary format designed for practical stemming workflows.
|
||||
Radixor uses a simple line-oriented dictionary format designed for practical stemming workflows. The textual source format is tab-separated values, meaning that columns are separated by the tab character.
|
||||
|
||||
Each logical line describes one canonical stem and zero or more known word variants that should reduce to that stem. The format is intentionally lightweight, easy to maintain in source control, and directly consumable both by the programmatic loader and by the CLI compiler.
|
||||
|
||||
@@ -9,16 +9,16 @@ Each logical line describes one canonical stem and zero or more known word varia
|
||||
Each non-empty logical line has the following shape:
|
||||
|
||||
```text
|
||||
<stem> <variant1> <variant2> <variant3> ...
|
||||
<stem> <variant1> <variant2> <variant3> ...
|
||||
```
|
||||
|
||||
The first token is interpreted as the **canonical stem**. Every following token on the same line is interpreted as a **known variant** belonging to that stem.
|
||||
The first column is interpreted as the **canonical stem**. Every following token on the same line is interpreted as a **known variant** belonging to that stem.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
```
|
||||
|
||||
In this example:
|
||||
@@ -30,7 +30,7 @@ In this example:
|
||||
|
||||
When a dictionary is loaded through `StemmerPatchTrieLoader`, the loader processes each parsed line as follows:
|
||||
|
||||
1. the first token becomes the canonical stem,
|
||||
1. the first column becomes the canonical stem,
|
||||
2. every following token is treated as a variant,
|
||||
3. each variant is converted into a patch command that transforms the variant into the stem,
|
||||
4. if `storeOriginal` is enabled, the stem itself is also inserted using the canonical no-op patch command.
|
||||
@@ -52,21 +52,23 @@ Whether such a line is operationally useful depends on how the dictionary is loa
|
||||
- if `storeOriginal` is enabled, the stem itself is inserted as a no-op mapping,
|
||||
- if `storeOriginal` is disabled, the line contributes no explicit variant mappings.
|
||||
|
||||
## Whitespace rules
|
||||
## Column and whitespace rules
|
||||
|
||||
Tokens are separated by whitespace. Leading and trailing whitespace is ignored.
|
||||
Columns are separated by the tab character. Leading and trailing whitespace around each column is ignored.
|
||||
|
||||
These lines are equivalent:
|
||||
This is the canonical form:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
run running runs ran
|
||||
```
|
||||
|
||||
This is also accepted because the surrounding padding is removed before the item is processed:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
run running runs ran
|
||||
```
|
||||
|
||||
Tabs and repeated spaces are both accepted because tokenization is whitespace-based.
|
||||
Embedded whitespace inside one dictionary item is currently not supported. A stem or variant such as `new york` therefore cannot yet be represented as one usable dictionary item in the textual source format. Such items are ignored during parsing and reported through a warning-level log entry together with the physical line number, the stem, and the ignored items from that line.
|
||||
|
||||
## Empty lines
|
||||
|
||||
@@ -75,9 +77,9 @@ Empty lines are ignored.
|
||||
Example:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
run running runs ran
|
||||
|
||||
connect connected connecting
|
||||
connect connected connecting
|
||||
```
|
||||
|
||||
The blank line between entries has no effect.
|
||||
@@ -96,8 +98,8 @@ The earliest occurrence of either marker terminates the logical content of the l
|
||||
Examples:
|
||||
|
||||
```text
|
||||
run running runs ran # English verb forms
|
||||
connect connected connecting // Common derived forms
|
||||
run running runs ran # English verb forms
|
||||
connect connected connecting // Common derived forms
|
||||
```
|
||||
|
||||
This is also valid:
|
||||
@@ -109,31 +111,37 @@ This is also valid:
|
||||
|
||||
## Case normalization
|
||||
|
||||
Input lines are normalized to lower case using `Locale.ROOT` before tokenization is processed into dictionary entries.
|
||||
Input-line case normalization is controlled by `CaseProcessingMode`; by default the parser uses `LOWERCASE_WITH_LOCALE_ROOT` before tab-separated columns are processed into dictionary entries.
|
||||
|
||||
That means dictionary authors should treat the format as **case-insensitive at load time**. If a file contains uppercase or mixed-case tokens, they will be normalized during parsing.
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
Run Running Runs Ran
|
||||
Run Running Runs Ran
|
||||
```
|
||||
|
||||
is processed the same way as:
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
run running runs ran
|
||||
```
|
||||
|
||||
## Character set and practical convention
|
||||
## Character set, compression, and normalization
|
||||
|
||||
Dictionary files are read as UTF-8 text.
|
||||
Dictionary files are read as UTF-8 text. Files loaded through `StemmerPatchTrieLoader.load(Path, ...)` may be either plain UTF-8 text or GZip-compressed UTF-8 text; the loader detects GZip input from the stream header instead of relying on the file extension. Bundled dictionaries are stored as GZip resources and are decoded as UTF-8 after decompression.
|
||||
|
||||
From the perspective of the parser and the stemming algorithm, the format is not restricted to plain ASCII tokens. The parser accepts ordinary Java `String` data, and the trie itself works with general character sequences rather than with an ASCII-only internal model. In principle, this means the system could process diacritic and non-diacritic forms alike, and it could also store forms with inconsistently used diacritics.
|
||||
The parser and trie are not restricted to ASCII. Dictionary items are ordinary Java `String` values, and trie traversal works over Java `char` sequences. This supports Latin-script data with diacritics, Cyrillic data, Hebrew, Persian, Yiddish, and other scripts represented in UTF-8, subject to the normal Java `String` model and the project’s traversal configuration.
|
||||
|
||||
In practice, however, the format is currently best understood as **primarily intended for classical basic ASCII lexical input**, especially in the traditional stemming style where language data is normalized into plain characters in the ASCII range up to character code 127. This convention is particularly relevant for languages whose original orthography includes diacritics but whose stemming dictionaries are commonly maintained in normalized non-diacritic form.
|
||||
Case normalization is controlled by `CaseProcessingMode`. The default `LOWERCASE_WITH_LOCALE_ROOT` mode lowercases the line before columns are split into dictionary items. `AS_IS` preserves the original casing.
|
||||
|
||||
Future versions may expand the documentation and operational guidance for dictionaries that intentionally preserve diacritics. At present, that workflow is not the primary documented use case, not because the algorithm fundamentally forbids it, but because a concrete project requirement for such support has not yet emerged.
|
||||
Diacritic normalization is controlled at trie-build and lookup time by `DiacriticProcessingMode`:
|
||||
|
||||
- `AS_IS` preserves dictionary and lookup keys exactly after case handling,
|
||||
- `REMOVE` strips supported diacritics and common Latin ligatures on both insertion and lookup paths,
|
||||
- `AS_IS_AND_STRIPPED_FALLBACK` is declared in the public model but is not implemented yet and raises `UnsupportedOperationException`.
|
||||
|
||||
For reliable production behavior, choose one normalization policy deliberately and apply it consistently. Normalized ASCII dictionaries remain a practical convention for some legacy stemming data, but they are not a format requirement.
|
||||
|
||||
## Distinct stem and variant semantics
|
||||
|
||||
@@ -142,8 +150,8 @@ The format expresses a one-line grouping of forms under a canonical stem. It doe
|
||||
For example:
|
||||
|
||||
```text
|
||||
axis axes
|
||||
axe axes
|
||||
axis axes
|
||||
axe axes
|
||||
```
|
||||
|
||||
These are simply two independent lines. If both contribute mappings for the same surface form, the compiled trie may later expose one or more candidate patch commands depending on the accumulated local counts and the selected reduction mode.
|
||||
@@ -163,35 +171,35 @@ As a result, repeating the same mapping is not just redundant text. It can influ
|
||||
### Simple English example
|
||||
|
||||
```text
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
build building builds built
|
||||
run running runs ran
|
||||
connect connected connecting connection
|
||||
build building builds built
|
||||
```
|
||||
|
||||
### Dictionary with remarks
|
||||
|
||||
```text
|
||||
run running runs ran # canonical verb family
|
||||
connect connected connecting // derived forms
|
||||
build building builds built
|
||||
run running runs ran # canonical verb family
|
||||
connect connected connecting // derived forms
|
||||
build building builds built
|
||||
```
|
||||
|
||||
### Stem-only entries
|
||||
|
||||
```text
|
||||
run
|
||||
connect connected connecting
|
||||
connect connected connecting
|
||||
build
|
||||
```
|
||||
|
||||
### Mixed case input
|
||||
|
||||
```text
|
||||
Run Running Runs Ran
|
||||
CONNECT Connected Connecting
|
||||
Run Running Runs Ran
|
||||
CONNECT Connected Connecting
|
||||
```
|
||||
|
||||
This is accepted, but it is normalized to lower case during parsing.
|
||||
This is accepted. Under the default `LOWERCASE_WITH_LOCALE_ROOT` mode it is normalized to lower case during parsing; under `AS_IS` it is preserved.
|
||||
|
||||
## Format limitations
|
||||
|
||||
@@ -204,7 +212,7 @@ The current dictionary format intentionally stays minimal:
|
||||
- no explicit ambiguity syntax,
|
||||
- no sectioning or nested structure.
|
||||
|
||||
Each token is simply a whitespace-delimited word form after remark stripping and lowercasing.
|
||||
Each dictionary item is simply one tab-separated word form after remark stripping and the configured case and diacritic normalization.
|
||||
|
||||
## Authoring guidance
|
||||
|
||||
@@ -216,7 +224,7 @@ For reliable results, keep dictionaries:
|
||||
- encoded in UTF-8,
|
||||
- easy to audit in plain text form.
|
||||
|
||||
For most current deployments, it is sensible to keep dictionary content in normalized basic ASCII form unless there is a clear requirement to preserve diacritics end-to-end.
|
||||
For most deployments, it is sensible to choose either preserved UTF-8 forms or a normalized ASCII/diacritic-stripped convention and keep that choice consistent across dictionary authoring, compilation, and runtime lookup.
|
||||
|
||||
## Relationship to other documentation
|
||||
|
||||
|
||||
@@ -87,3 +87,20 @@ This model works especially well when domain-specific extensions are added in la
|
||||
|
||||
- [Loading and Building Stemmers](programmatic-loading-and-building.md)
|
||||
- [Querying and Ambiguity Handling](programmatic-querying-and-ambiguity.md)
|
||||
|
||||
|
||||
## Inspecting persisted metadata
|
||||
|
||||
After loading a compiled artifact, applications can inspect the persisted build descriptor directly:
|
||||
|
||||
```java
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.loadBinary("build/stemmers/cs_cz.dat.gz");
|
||||
final TrieMetadata metadata = trie.metadata();
|
||||
|
||||
System.out.println(metadata.formatVersion());
|
||||
System.out.println(metadata.traversalDirection());
|
||||
System.out.println(metadata.reductionSettings().reductionMode());
|
||||
System.out.println(metadata.diacriticProcessingMode());
|
||||
```
|
||||
|
||||
This is especially useful when a deployment manages multiple artifacts compiled under different traversal or reduction regimes.
|
||||
|
||||
@@ -21,7 +21,7 @@ public final class BundledLanguageExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ The `storeOriginal` flag controls whether the canonical stem is inserted as a no
|
||||
|
||||
## Load a textual dictionary
|
||||
|
||||
Loading from a dictionary file follows the same preparation model as bundled resources, but the source comes from your own file or path. Each non-empty logical line starts with the stem and may contain zero or more variants. Input is normalized to lower case using `Locale.ROOT`, and trailing remarks introduced by `#` or `//` are ignored.
|
||||
Loading from a dictionary file follows the same preparation model as bundled resources, but the source comes from your own file or path. The input may be plain UTF-8 text or GZip-compressed UTF-8 text; the loader detects GZip data from the stream header. The textual format is tab-separated values, meaning that columns are separated by the tab character. Each non-empty logical line starts with the stem column and may contain zero or more variant columns. Input case normalization is controlled by `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), trailing remarks introduced by `#` or `//` are ignored, and dictionary items containing embedded whitespace are currently ignored with warning-level diagnostics.
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
@@ -51,7 +51,7 @@ public final class LoadTextDictionaryExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
Path.of("data", "stemmer.txt"),
|
||||
Path.of("data", "stemmer.tsv"),
|
||||
true,
|
||||
ReductionSettings.withDefaults(
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS));
|
||||
@@ -59,6 +59,8 @@ public final class LoadTextDictionaryExample {
|
||||
}
|
||||
```
|
||||
|
||||
Additional `StemmerPatchTrieLoader.load(...)` overloads let callers provide explicit `WordTraversalDirection`, `CaseProcessingMode`, `DiacriticProcessingMode`, or a complete `TrieMetadata` instance. Use those overloads when a custom dictionary must be compiled with forward traversal for right-to-left languages, case-sensitive keys, or diacritic stripping.
|
||||
|
||||
## Load a compiled binary artifact
|
||||
|
||||
Binary loading is typically the preferred runtime path because it avoids reparsing the textual source and skips the preparation step entirely.
|
||||
@@ -83,7 +85,7 @@ public final class LoadBinaryExample {
|
||||
}
|
||||
```
|
||||
|
||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression.
|
||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
|
||||
|
||||
## Build directly with a mutable builder
|
||||
|
||||
@@ -108,7 +110,7 @@ public final class BuilderExample {
|
||||
final FrequencyTrie.Builder<String> builder =
|
||||
new FrequencyTrie.Builder<>(String[]::new, settings);
|
||||
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
builder.put("running", encoder.encode("running", "run"));
|
||||
builder.put("runs", encoder.encode("runs", "run"));
|
||||
|
||||
@@ -15,7 +15,7 @@ A compiled stemmer can be obtained in three common ways.
|
||||
|
||||
### Use a bundled language dictionary
|
||||
|
||||
Radixor ships with bundled dictionaries for a set of supported languages. These resources are line-oriented dictionaries stored with the library and compiled into a `FrequencyTrie<String>` when loaded. The loader can also store the canonical stem itself as a no-op patch command.
|
||||
Radixor ships with bundled dictionaries for a set of supported languages. These resources are line-oriented dictionaries stored with the library and compiled into a `FrequencyTrie<String>` when loaded. The loader can also store the canonical stem itself as a no-op patch command. Compiled trie artifacts now persist self-describing metadata, including the traversal direction and compilation reduction settings used to build the artifact.
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
@@ -32,7 +32,7 @@ public final class BundledStemmerExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
@@ -69,7 +69,7 @@ public final class LoadBinaryStemmerExample {
|
||||
|
||||
### Build or extend a stemmer from dictionary data
|
||||
|
||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The parser lowercases input with `Locale.ROOT`, ignores leading and trailing whitespace, and supports line remarks introduced by `#` or `//`.
|
||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.
|
||||
|
||||
This path is also relevant when you extend an existing compiled stemmer with additional domain-specific entries and rebuild a new compact artifact.
|
||||
|
||||
@@ -104,7 +104,7 @@ public final class SingleStemExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
@@ -202,3 +202,10 @@ Dictionary compilation is usually a one-time preparation step and is generally f
|
||||
- [CLI compilation](cli-compilation.md)
|
||||
- [Built-in languages](built-in-languages.md)
|
||||
- [Architecture and reduction](architecture-and-reduction.md)
|
||||
|
||||
|
||||
## Persisted trie metadata
|
||||
|
||||
Every compiled trie artifact stores a `TrieMetadata` descriptor together with the immutable trie payload. That metadata currently records the binary format version, the `WordTraversalDirection`, the `ReductionSettings` used during compilation, the declared `DiacriticProcessingMode`, and the selected `CaseProcessingMode`. Traversal, case processing, and diacritic processing are applied during runtime lookup (`get`, `getAll`), and case/diacritic processing are also applied during dictionary insertion when a trie is built.
|
||||
|
||||
`DiacriticProcessingMode.AS_IS` keeps dictionary keys and lookup keys unchanged. `DiacriticProcessingMode.REMOVE` strips diacritics from dictionary keys and lookup keys (for Czech diacritics and broad European Latin-script variants). `DiacriticProcessingMode.AS_IS_AND_STRIPPED_FALLBACK` is currently not supported and raises an `UnsupportedOperationException`.
|
||||
|
||||
@@ -17,3 +17,6 @@ pomScmDeveloperConnection=scm:git:ssh://git@github.com/leogalambos/Radixor.git
|
||||
|
||||
pomLicenseName=BSD-3-Clause
|
||||
pomLicenseUrl=https://spdx.org/licenses/BSD-3-Clause.html
|
||||
|
||||
pomStemmerDataLicenseName=Stemmer Data License Policy
|
||||
pomStemmerDataLicenseUrl=https://github.com/leogalambos/Radixor/blob/main/LICENSE-stemmer-data
|
||||
|
||||
@@ -13,6 +13,12 @@ def pomScmDeveloperConnection = providers.gradleProperty('pomScmDeveloperConnect
|
||||
def pomLicenseName = providers.gradleProperty('pomLicenseName').orNull
|
||||
def pomLicenseUrl = providers.gradleProperty('pomLicenseUrl').orNull
|
||||
def pomLicenseDistribution = providers.gradleProperty('pomLicenseDistribution').orElse('repo').get()
|
||||
def pomStemmerDataLicenseName = providers.gradleProperty('pomStemmerDataLicenseName')
|
||||
.orElse('Stemmer Data License Policy')
|
||||
.get()
|
||||
def pomStemmerDataLicenseUrl = providers.gradleProperty('pomStemmerDataLicenseUrl')
|
||||
.orElse('https://github.com/leogalambos/Radixor/blob/main/LICENSE-stemmer-data')
|
||||
.get()
|
||||
def pomDeveloperId = providers.gradleProperty('pomDeveloperId').orElse('egothor').get()
|
||||
def pomDeveloperName = providers.gradleProperty('pomDeveloperName').orElse('Leo Galambos').get()
|
||||
def pomDeveloperEmail = providers.gradleProperty('pomDeveloperEmail').orElse('egothor@gmail.com').get()
|
||||
@@ -45,6 +51,11 @@ publishing {
|
||||
url = pomLicenseUrl
|
||||
distribution = pomLicenseDistribution
|
||||
}
|
||||
license {
|
||||
name = pomStemmerDataLicenseName
|
||||
url = pomStemmerDataLicenseUrl
|
||||
distribution = pomLicenseDistribution
|
||||
}
|
||||
}
|
||||
|
||||
developers {
|
||||
@@ -93,6 +104,8 @@ tasks.register('validateReleaseMetadata') {
|
||||
if (pomScmDeveloperConnection == null || pomScmDeveloperConnection.isBlank()) missing.add('pomScmDeveloperConnection')
|
||||
if (pomLicenseName == null || pomLicenseName.isBlank()) missing.add('pomLicenseName')
|
||||
if (pomLicenseUrl == null || pomLicenseUrl.isBlank()) missing.add('pomLicenseUrl')
|
||||
if (pomStemmerDataLicenseName == null || pomStemmerDataLicenseName.isBlank()) missing.add('pomStemmerDataLicenseName')
|
||||
if (pomStemmerDataLicenseUrl == null || pomStemmerDataLicenseUrl.isBlank()) missing.add('pomStemmerDataLicenseUrl')
|
||||
if (signingKey == null || signingKey.isBlank()) missing.add('pomSigningKey / SIGNING_KEY')
|
||||
if (signingPassword == null || signingPassword.isBlank()) missing.add('pomSigningPassword / SIGNING_PASSWORD')
|
||||
|
||||
|
||||
@@ -1,10 +1,27 @@
|
||||
import org.gradle.plugins.ide.eclipse.model.SourceFolder
|
||||
|
||||
|
||||
def snowballVersion = '3.0.1'
|
||||
def snowballArchiveName = "libstemmer_java-${snowballVersion}.tar.gz"
|
||||
def snowballDistributionDirectoryName = "libstemmer_java-${snowballVersion}"
|
||||
def snowballRootRelativePath = 'third-party/snowball'
|
||||
def snowballSourceRelativePath = "${snowballRootRelativePath}/source"
|
||||
def snowballJavaSourceRelativePath = "${snowballSourceRelativePath}/${snowballDistributionDirectoryName}/java"
|
||||
def snowballDownloadUrl = "https://snowballstem.org/dist/${snowballArchiveName}"
|
||||
def snowballDownloadFile = layout.buildDirectory.file("third-party/snowball/${snowballArchiveName}")
|
||||
def snowballExtractDirectory = layout.buildDirectory.dir('third-party/snowball/source')
|
||||
def snowballJavaSourceDirectory = layout.buildDirectory.dir(
|
||||
"third-party/snowball/source/libstemmer_java-${snowballVersion}/java")
|
||||
def snowballDownloadFile = layout.buildDirectory.file("${snowballRootRelativePath}/${snowballArchiveName}")
|
||||
def snowballExtractDirectory = layout.buildDirectory.dir(snowballSourceRelativePath)
|
||||
def snowballJavaSourceDirectory = layout.buildDirectory.dir(snowballJavaSourceRelativePath)
|
||||
def snowballJavaSourceClasspathPath = provider {
|
||||
project.relativePath(snowballJavaSourceDirectory.get().asFile)
|
||||
}
|
||||
def snowballEclipseClasspathAttributes = [
|
||||
gradle_scope : 'jmh',
|
||||
gradle_used_by_scope: 'jmh',
|
||||
test : 'true'
|
||||
]
|
||||
def isAbsoluteClasspathPath = { String path ->
|
||||
path.startsWith('/') || path ==~ /^[A-Za-z]:[\\\/].*/
|
||||
}
|
||||
|
||||
tasks.register('downloadSnowballJava') {
|
||||
group = 'build setup'
|
||||
@@ -46,4 +63,31 @@ sourceSets {
|
||||
|
||||
tasks.named('compileJmhJava') {
|
||||
dependsOn(tasks.named('extractSnowballJava'))
|
||||
}
|
||||
}
|
||||
|
||||
eclipse {
|
||||
classpath {
|
||||
file {
|
||||
whenMerged { classpath ->
|
||||
String generatedSnowballPath = snowballJavaSourceClasspathPath.get()
|
||||
String modelSnowballPath = snowballJavaSourceRelativePath
|
||||
|
||||
classpath.entries.removeAll { entry ->
|
||||
entry.hasProperty('path') && (
|
||||
entry.path == generatedSnowballPath ||
|
||||
entry.path == modelSnowballPath ||
|
||||
isAbsoluteClasspathPath(entry.path)
|
||||
)
|
||||
}
|
||||
|
||||
SourceFolder snowballEntry = new SourceFolder(generatedSnowballPath, null)
|
||||
snowballEntry.output = 'bin/jmh'
|
||||
snowballEclipseClasspathAttributes.each { String name, String value ->
|
||||
snowballEntry.entryAttributes[name] = value
|
||||
}
|
||||
|
||||
classpath.entries.add(snowballEntry)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ final class BenchmarkCorpusSupport {
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, reductionSettings);
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
StemmerDictionaryParser.parse(
|
||||
new StringReader(corpusText),
|
||||
|
||||
@@ -59,8 +59,7 @@ import org.tartarus.snowball.ext.porterStemmer;
|
||||
* The benchmark processes the same deterministic token array with:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>Radixor using bundled
|
||||
* {@link StemmerPatchTrieLoader.Language#US_UK_PROFI}</li>
|
||||
* <li>Radixor using bundled {@link StemmerPatchTrieLoader.Language#US_UK}</li>
|
||||
* <li>Snowball original Porter stemmer</li>
|
||||
* <li>Snowball English stemmer, commonly referred to as Porter2</li>
|
||||
* </ul>
|
||||
@@ -106,7 +105,7 @@ public class EnglishStemmerComparisonBenchmark {
|
||||
@Setup(Level.Trial)
|
||||
public void setUp() throws IOException {
|
||||
this.tokens = EnglishComparisonCorpus.createTokens(this.familyCount);
|
||||
this.radixorTrie = StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK_PROFI, true,
|
||||
this.radixorTrie = StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK, true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
}
|
||||
}
|
||||
|
||||
55
src/main/java/org/egothor/stemmer/CaseProcessingMode.java
Normal file
55
src/main/java/org/egothor/stemmer/CaseProcessingMode.java
Normal file
@@ -0,0 +1,55 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Defines how dictionary items are normalized with respect to letter casing.
|
||||
*
|
||||
* <p>
|
||||
* The mode is applied while parsing dictionary sources and can be persisted in
|
||||
* trie metadata so that compiled artifacts remain self-describing.
|
||||
*/
|
||||
public enum CaseProcessingMode {
|
||||
|
||||
/**
|
||||
* Preserves input character casing exactly as provided by the dictionary
|
||||
* source.
|
||||
*/
|
||||
AS_IS,
|
||||
|
||||
/**
|
||||
* Normalizes all dictionary content to lower case using
|
||||
* {@link Locale#ROOT}.
|
||||
*/
|
||||
LOWERCASE_WITH_LOCALE_ROOT
|
||||
}
|
||||
@@ -61,6 +61,8 @@ import java.util.logging.Logger;
|
||||
* --output <file>
|
||||
* --reduction-mode <mode>
|
||||
* [--store-original]
|
||||
* [--right-to-left]
|
||||
* [--case-processing-mode <mode>]
|
||||
* [--dominant-winner-min-percent <1..100>]
|
||||
* [--dominant-winner-over-second-ratio <1..n>]
|
||||
* [--overwrite]
|
||||
@@ -149,8 +151,10 @@ public final class Compile {
|
||||
final ReductionSettings reductionSettings = new ReductionSettings(arguments.reductionMode(),
|
||||
arguments.dominantWinnerMinPercent(), arguments.dominantWinnerOverSecondRatio());
|
||||
|
||||
final WordTraversalDirection traversalDirection = arguments.rightToLeft() ? WordTraversalDirection.FORWARD
|
||||
: WordTraversalDirection.BACKWARD;
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(arguments.inputFile(), arguments.storeOriginal(),
|
||||
reductionSettings);
|
||||
reductionSettings, traversalDirection, arguments.caseProcessingMode());
|
||||
|
||||
final Path outputFile = arguments.outputFile();
|
||||
final Path parent = outputFile.toAbsolutePath().getParent();
|
||||
@@ -166,11 +170,11 @@ public final class Compile {
|
||||
|
||||
if (LOGGER.isLoggable(Level.INFO)) {
|
||||
LOGGER.log(Level.INFO,
|
||||
"Compiled dictionary {0} to {1} using mode {2}, storeOriginal={3}, dominantWinnerMinPercent={4}, dominantWinnerOverSecondRatio={5}.",
|
||||
"Compiled dictionary {0} to {1} using mode {2}, storeOriginal={3}, rightToLeft={4}, caseProcessingMode={5}, dominantWinnerMinPercent={6}, dominantWinnerOverSecondRatio={7}.",
|
||||
new Object[] { arguments.inputFile().toAbsolutePath().toString(),
|
||||
arguments.outputFile().toAbsolutePath().toString(), arguments.reductionMode().name(),
|
||||
arguments.storeOriginal(), arguments.dominantWinnerMinPercent(),
|
||||
arguments.dominantWinnerOverSecondRatio() });
|
||||
arguments.storeOriginal(), arguments.rightToLeft(), arguments.caseProcessingMode(),
|
||||
arguments.dominantWinnerMinPercent(), arguments.dominantWinnerOverSecondRatio() });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,10 +188,28 @@ public final class Compile {
|
||||
System.err.println(" --output <file> \\");
|
||||
System.err.println(" --reduction-mode <mode> \\");
|
||||
System.err.println(" [--store-original] \\");
|
||||
System.err.println(" [--case-processing-mode <mode>] \\");
|
||||
System.err.println(" [--dominant-winner-min-percent <1..100>] \\");
|
||||
System.err.println(" [--dominant-winner-over-second-ratio <1..n>] \\");
|
||||
System.err.println(" [--overwrite]");
|
||||
System.err.println();
|
||||
System.err.println("Options:");
|
||||
System.err.println(" --store-original");
|
||||
System.err.println(" Inserts each canonical stem itself using the no-operation patch.");
|
||||
System.err.println(" --right-to-left");
|
||||
System.err.println(" Uses forward word traversal for right-to-left languages.");
|
||||
System.err.println(" In this mode, trie keys are constructed from the logical beginning");
|
||||
System.err.println(" of the stored word form and patch commands are encoded likewise.");
|
||||
System.err.println(" --overwrite");
|
||||
System.err.println(" Replaces the target file when it already exists.");
|
||||
System.err.println(" --case-processing-mode");
|
||||
System.err.println(" Controls whether dictionary input is lowercased or preserved as-is.");
|
||||
System.err.println();
|
||||
System.err.println("Supported case processing modes:");
|
||||
for (CaseProcessingMode mode : CaseProcessingMode.values()) {
|
||||
System.err.println(" " + mode.name());
|
||||
}
|
||||
System.err.println();
|
||||
System.err.println("Supported reduction modes:");
|
||||
for (ReductionMode mode : ReductionMode.values()) {
|
||||
System.err.println(" " + mode.name());
|
||||
@@ -240,15 +262,20 @@ public final class Compile {
|
||||
* @param outputFile output compressed trie file
|
||||
* @param reductionMode subtree reduction mode
|
||||
* @param storeOriginal whether original stems are stored
|
||||
* @param rightToLeft whether dictionary compilation should
|
||||
* use forward traversal on stored word
|
||||
* forms
|
||||
* @param dominantWinnerMinPercent dominant winner minimum percent
|
||||
* @param dominantWinnerOverSecondRatio dominant winner over second ratio
|
||||
* @param caseProcessingMode dictionary case processing mode
|
||||
* @param overwrite whether an existing output may be
|
||||
* replaced
|
||||
* @param help whether usage help was requested
|
||||
*/
|
||||
@SuppressWarnings("PMD.LongVariable")
|
||||
private record Arguments(Path inputFile, Path outputFile, ReductionMode reductionMode, boolean storeOriginal,
|
||||
int dominantWinnerMinPercent, int dominantWinnerOverSecondRatio, boolean overwrite, boolean help) {
|
||||
boolean rightToLeft, int dominantWinnerMinPercent, int dominantWinnerOverSecondRatio,
|
||||
CaseProcessingMode caseProcessingMode, boolean overwrite, boolean help) {
|
||||
|
||||
/**
|
||||
* Parses raw command-line arguments.
|
||||
@@ -264,8 +291,10 @@ public final class Compile {
|
||||
Path outputFile = null;
|
||||
ReductionMode reductionMode = null;
|
||||
boolean storeOriginal = false;
|
||||
boolean rightToLeft = false;
|
||||
boolean overwrite = false;
|
||||
boolean help = false;
|
||||
CaseProcessingMode caseProcessingMode = CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||
int dominantWinnerMinPercent = ReductionSettings.DEFAULT_DOMINANT_WINNER_MIN_PERCENT;
|
||||
int dominantWinnerOverSecondRatio = ReductionSettings.DEFAULT_DOMINANT_WINNER_OVER_SECOND_RATIO;
|
||||
|
||||
@@ -286,6 +315,10 @@ public final class Compile {
|
||||
overwrite = true;
|
||||
break;
|
||||
|
||||
case "--right-to-left":
|
||||
rightToLeft = true;
|
||||
break;
|
||||
|
||||
case "--input":
|
||||
inputFile = Path.of(requireValue(arguments, ++index, "--input"));
|
||||
break;
|
||||
@@ -310,6 +343,10 @@ public final class Compile {
|
||||
requireValue(arguments, ++index, "--dominant-winner-over-second-ratio"),
|
||||
"--dominant-winner-over-second-ratio");
|
||||
break;
|
||||
case "--case-processing-mode":
|
||||
caseProcessingMode = CaseProcessingMode.valueOf(
|
||||
requireValue(arguments, ++index, "--case-processing-mode").toUpperCase(Locale.ROOT));
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown argument: " + argument);
|
||||
@@ -317,8 +354,8 @@ public final class Compile {
|
||||
}
|
||||
|
||||
if (help) {
|
||||
return new Arguments(inputFile, outputFile, reductionMode, storeOriginal, dominantWinnerMinPercent,
|
||||
dominantWinnerOverSecondRatio, overwrite, true);
|
||||
return new Arguments(inputFile, outputFile, reductionMode, storeOriginal, rightToLeft,
|
||||
dominantWinnerMinPercent, dominantWinnerOverSecondRatio, caseProcessingMode, overwrite, true);
|
||||
}
|
||||
|
||||
if (inputFile == null) {
|
||||
@@ -331,8 +368,8 @@ public final class Compile {
|
||||
throw new IllegalArgumentException("Missing required argument --reduction-mode.");
|
||||
}
|
||||
|
||||
return new Arguments(inputFile, outputFile, reductionMode, storeOriginal, dominantWinnerMinPercent,
|
||||
dominantWinnerOverSecondRatio, overwrite, false);
|
||||
return new Arguments(inputFile, outputFile, reductionMode, storeOriginal, rightToLeft,
|
||||
dominantWinnerMinPercent, dominantWinnerOverSecondRatio, caseProcessingMode, overwrite, false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
/**
|
||||
* Defines how dictionary loading and trie traversal should treat diacritics.
|
||||
*
|
||||
* <p>
|
||||
* The selected mode is applied independently from other normalization modes
|
||||
* (for example {@link CaseProcessingMode}). This means case normalization and
|
||||
* diacritic normalization can be combined freely and each keeps its own
|
||||
* semantics.
|
||||
* </p>
|
||||
*/
|
||||
public enum DiacriticProcessingMode {
|
||||
|
||||
/**
|
||||
* Preserves dictionary entries and lookup keys exactly as provided.
|
||||
*/
|
||||
AS_IS,
|
||||
|
||||
/**
|
||||
* Removes diacritics from dictionary entries before trie construction and
|
||||
* removes diacritics from lookup keys before traversal.
|
||||
*/
|
||||
REMOVE,
|
||||
|
||||
/**
|
||||
* Planned dual-path mode where lookup may continue along both the original
|
||||
* diacritic edge and a normalized non-diacritic alternative.
|
||||
*
|
||||
* <p>
|
||||
* This mode is currently not supported and using it triggers
|
||||
* {@link UnsupportedOperationException}.
|
||||
* </p>
|
||||
*/
|
||||
AS_IS_AND_STRIPPED_FALLBACK
|
||||
}
|
||||
197
src/main/java/org/egothor/stemmer/DiacriticStripper.java
Normal file
197
src/main/java/org/egothor/stemmer/DiacriticStripper.java
Normal file
@@ -0,0 +1,197 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.text.Normalizer;
|
||||
import java.text.Normalizer.Form;
|
||||
|
||||
/**
|
||||
* Utility that strips diacritics from text for diacritic-insensitive trie
|
||||
* storage and lookup.
|
||||
*/
|
||||
final class DiacriticStripper {
|
||||
|
||||
/**
|
||||
* Direct single-character replacement table.
|
||||
*/
|
||||
private static final char[] DIRECT_REPLACEMENTS = new char[Character.MAX_VALUE + 1];
|
||||
|
||||
static {
|
||||
registerSingle("áàâäãåāăąǎȁȃạảấầẩẫậắằẳẵặ", 'a');
|
||||
registerSingle("ÁÀÂÄÃÅĀĂĄǍȀȂẠẢẤẦẨẪẬẮẰẲẴẶ", 'A');
|
||||
registerSingle("çćĉċč", 'c');
|
||||
registerSingle("ÇĆĈĊČ", 'C');
|
||||
registerSingle("ďđḍ", 'd');
|
||||
registerSingle("ĎĐḌ", 'D');
|
||||
registerSingle("éèêëēĕėęěȅȇẹẻẽếềểễệ", 'e');
|
||||
registerSingle("ÉÈÊËĒĔĖĘĚȄȆẸẺẼẾỀỂỄỆ", 'E');
|
||||
registerSingle("ğĝġģǧ", 'g');
|
||||
registerSingle("ĞĜĠĢǦ", 'G');
|
||||
registerSingle("ĥħ", 'h');
|
||||
registerSingle("ĤĦ", 'H');
|
||||
registerSingle("íìîïĩīĭįıǐȉȋịỉ", 'i');
|
||||
registerSingle("ÍÌÎÏĨĪĬĮİǏȈȊỊỈ", 'I');
|
||||
registerSingle("ĵ", 'j');
|
||||
registerSingle("Ĵ", 'J');
|
||||
registerSingle("ķǩ", 'k');
|
||||
registerSingle("ĶǨ", 'K');
|
||||
registerSingle("ĺļľŀł", 'l');
|
||||
registerSingle("ĹĻĽĿŁ", 'L');
|
||||
registerSingle("ñńņňʼnŋ", 'n');
|
||||
registerSingle("ÑŃŅŇŊ", 'N');
|
||||
registerSingle("óòôöõōŏőǒȍȏọỏốồổỗộớờởỡợø", 'o');
|
||||
registerSingle("ÓÒÔÖÕŌŎŐǑȌȎỌỎỐỒỔỖỘỚỜỞỠỢØ", 'O');
|
||||
registerSingle("ŕŗř", 'r');
|
||||
registerSingle("ŔŖŘ", 'R');
|
||||
registerSingle("śŝşšș", 's');
|
||||
registerSingle("ŚŜŞŠȘ", 'S');
|
||||
registerSingle("ťţŧț", 't');
|
||||
registerSingle("ŤŢŦȚ", 'T');
|
||||
registerSingle("úùûüũūŭůűųǔȕȗụủứừửữự", 'u');
|
||||
registerSingle("ÚÙÛÜŨŪŬŮŰŲǓȔȖỤỦỨỪỬỮỰ", 'U');
|
||||
registerSingle("ýÿŷỳỵỷỹ", 'y');
|
||||
registerSingle("ÝŶŸỲỴỶỸ", 'Y');
|
||||
registerSingle("źżž", 'z');
|
||||
registerSingle("ŹŻŽ", 'Z');
|
||||
registerSingle("þ", 't');
|
||||
registerSingle("Þ", 'T');
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility class.
|
||||
*/
|
||||
private DiacriticStripper() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes supported diacritic marks and common Latin ligatures from the supplied
|
||||
* text.
|
||||
*
|
||||
* <p>
|
||||
* The method returns the original {@link String} instance when no replacement is
|
||||
* required, avoiding an unnecessary allocation on the common ASCII path.
|
||||
* </p>
|
||||
*
|
||||
* @param input text to normalize
|
||||
* @return normalized text, or {@code input} itself when it is already unchanged
|
||||
*/
|
||||
/* default */ static String strip(final String input) {
|
||||
StringBuilder normalized = null;
|
||||
|
||||
for (int index = 0; index < input.length(); index++) {
|
||||
final char source = input.charAt(index);
|
||||
final String replacement = replacementFor(source);
|
||||
|
||||
if (replacement == null) {
|
||||
if (normalized != null) {
|
||||
normalized.append(source);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (normalized == null) {
|
||||
normalized = new StringBuilder(input.length()); // NOPMD - invariant: only once
|
||||
normalized.append(input, 0, index);
|
||||
}
|
||||
normalized.append(replacement);
|
||||
}
|
||||
|
||||
if (normalized == null) {
|
||||
return input;
|
||||
}
|
||||
return normalized.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the replacement text for one non-ASCII character.
|
||||
*
|
||||
* @param source source character
|
||||
* @return replacement text, or {@code null} when the character should be kept
|
||||
* unchanged
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
private static String replacementFor(final char source) {
|
||||
if (source <= 0x007F) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final char mapped = DIRECT_REPLACEMENTS[source];
|
||||
if (mapped != '\0') {
|
||||
return String.valueOf(mapped);
|
||||
}
|
||||
|
||||
if (source == 'ß') {
|
||||
return "ss";
|
||||
}
|
||||
if (source == 'Æ') {
|
||||
return "AE";
|
||||
}
|
||||
if (source == 'æ') {
|
||||
return "ae";
|
||||
}
|
||||
if (source == 'Œ') {
|
||||
return "OE";
|
||||
}
|
||||
if (source == 'œ') {
|
||||
return "oe";
|
||||
}
|
||||
|
||||
final String decomposed = Normalizer.normalize(String.valueOf(source), Form.NFD);
|
||||
final StringBuilder ascii = new StringBuilder(decomposed.length());
|
||||
for (int index = 0; index < decomposed.length(); index++) {
|
||||
final char part = decomposed.charAt(index);
|
||||
if (Character.getType(part) == Character.NON_SPACING_MARK) {
|
||||
continue;
|
||||
}
|
||||
if (part <= 0x007F) {
|
||||
ascii.append(part);
|
||||
}
|
||||
}
|
||||
|
||||
if (ascii.length() == 0) {
|
||||
return null;
|
||||
}
|
||||
return ascii.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers one-character replacements for a set of source characters.
|
||||
*
|
||||
* @param sourceCharacters characters to replace
|
||||
* @param replacement replacement character
|
||||
*/
|
||||
private static void registerSingle(final String sourceCharacters, final char replacement) {
|
||||
for (int index = 0; index < sourceCharacters.length(); index++) {
|
||||
DIRECT_REPLACEMENTS[sourceCharacters.charAt(index)] = replacement;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ import java.util.Collections;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.IntFunction;
|
||||
@@ -86,6 +87,7 @@ import org.egothor.stemmer.trie.ReductionSignature;
|
||||
*
|
||||
* @param <V> value type
|
||||
*/
|
||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||
public final class FrequencyTrie<V> {
|
||||
|
||||
/**
|
||||
@@ -93,16 +95,6 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName());
|
||||
|
||||
/**
|
||||
* Binary format magic header.
|
||||
*/
|
||||
private static final int STREAM_MAGIC = 0x45475452;
|
||||
|
||||
/**
|
||||
* Binary format version.
|
||||
*/
|
||||
private static final int STREAM_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Factory used to create correctly typed arrays for {@link #getAll(String)}.
|
||||
*/
|
||||
@@ -113,16 +105,49 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private final CompiledNode<V> root;
|
||||
|
||||
/**
|
||||
* Metadata persisted together with this trie.
|
||||
*/
|
||||
private final TrieMetadata metadata;
|
||||
|
||||
/**
|
||||
* Binary format magic header.
|
||||
*/
|
||||
private static final int STREAM_MAGIC = 0x45475452;
|
||||
|
||||
/**
|
||||
* Binary format version.
|
||||
*/
|
||||
private static final int STREAM_VERSION = 5;
|
||||
|
||||
/**
|
||||
* Returns the current persisted binary stream format version.
|
||||
*
|
||||
* <p>
|
||||
* This method exists so other components can construct {@link TrieMetadata}
|
||||
* instances aligned with the currently written binary format without
|
||||
* duplicating constants.
|
||||
* </p>
|
||||
*
|
||||
* @return current trie stream format version
|
||||
*/
|
||||
public static int currentFormatVersion() {
|
||||
return STREAM_VERSION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new compiled trie instance.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param root compiled root node
|
||||
* @param metadata trie metadata describing lookup and persistence semantics
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
private FrequencyTrie(final IntFunction<V[]> arrayFactory, final CompiledNode<V> root) {
|
||||
private FrequencyTrie(final IntFunction<V[]> arrayFactory, final CompiledNode<V> root,
|
||||
final TrieMetadata metadata) {
|
||||
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
this.root = Objects.requireNonNull(root, "root");
|
||||
this.metadata = Objects.requireNonNull(metadata, "metadata");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -134,6 +159,10 @@ public final class FrequencyTrie<V> {
|
||||
* selected deterministically by shorter {@code toString()} value first, then by
|
||||
* lexicographically lower {@code toString()}, and finally by stable first-seen
|
||||
* order.
|
||||
*
|
||||
* <p>
|
||||
* The supplied key is normalized according to persisted
|
||||
* {@link TrieMetadata#caseProcessingMode()} before traversal.
|
||||
*
|
||||
* @param key key to resolve
|
||||
* @return most frequent value, or {@code null} if the key does not exist or no
|
||||
@@ -142,7 +171,7 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
public V get(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(key);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
return null;
|
||||
}
|
||||
@@ -162,6 +191,10 @@ public final class FrequencyTrie<V> {
|
||||
* <p>
|
||||
* The returned array is a defensive copy.
|
||||
*
|
||||
* <p>
|
||||
* The supplied key is normalized according to persisted
|
||||
* {@link TrieMetadata#caseProcessingMode()} before traversal.
|
||||
*
|
||||
* @param key key to resolve
|
||||
* @return all values stored at the addressed node, ordered by descending
|
||||
* frequency; returns an empty array if the key does not exist or no
|
||||
@@ -170,7 +203,7 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
public V[] getAll(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(key);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
return this.arrayFactory.apply(0);
|
||||
}
|
||||
@@ -201,7 +234,7 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
public List<ValueCount<V>> getEntries(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(key);
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
return List.of();
|
||||
}
|
||||
@@ -213,6 +246,29 @@ public final class FrequencyTrie<V> {
|
||||
return Collections.unmodifiableList(entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the logical key traversal direction used by this trie.
|
||||
*
|
||||
* <p>
|
||||
* The same direction must be used when reconstructing mutable builders or when
|
||||
* applying patch commands that were generated against keys stored in this trie.
|
||||
* </p>
|
||||
*
|
||||
* @return logical key traversal direction
|
||||
*/
|
||||
public WordTraversalDirection traversalDirection() {
|
||||
return this.metadata.traversalDirection();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns immutable persisted metadata associated with this trie.
|
||||
*
|
||||
* @return trie metadata
|
||||
*/
|
||||
public TrieMetadata metadata() {
|
||||
return this.metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the root node mainly for diagnostics and tests within the package.
|
||||
*
|
||||
@@ -262,6 +318,7 @@ public final class FrequencyTrie<V> {
|
||||
dataOutput.writeInt(STREAM_VERSION);
|
||||
dataOutput.writeInt(orderedNodes.size());
|
||||
dataOutput.writeInt(nodeIds.get(this.root));
|
||||
writeMetadata(dataOutput, this.metadata);
|
||||
|
||||
for (CompiledNode<V> node : orderedNodes) {
|
||||
writeNode(dataOutput, valueCodec, node, nodeIds);
|
||||
@@ -304,7 +361,7 @@ public final class FrequencyTrie<V> {
|
||||
}
|
||||
|
||||
final int version = dataInput.readInt();
|
||||
if (version != STREAM_VERSION) {
|
||||
if (version != 1 && version != 3 && version != 4 && version != STREAM_VERSION) {
|
||||
throw new IOException("Unsupported trie stream version: " + version);
|
||||
}
|
||||
|
||||
@@ -318,6 +375,8 @@ public final class FrequencyTrie<V> {
|
||||
throw new IOException("Invalid root node id: " + rootNodeId);
|
||||
}
|
||||
|
||||
final TrieMetadata metadata = readMetadata(dataInput, version);
|
||||
|
||||
final CompiledNode<V>[] nodes = readNodes(dataInput, arrayFactory, valueCodec, nodeCount);
|
||||
final CompiledNode<V> rootNode = nodes[rootNodeId];
|
||||
|
||||
@@ -325,7 +384,86 @@ public final class FrequencyTrie<V> {
|
||||
LOGGER.log(Level.FINE, "Read compiled trie with {0} canonical nodes.", nodeCount);
|
||||
}
|
||||
|
||||
return new FrequencyTrie<>(arrayFactory, rootNode);
|
||||
return new FrequencyTrie<>(arrayFactory, rootNode, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes persisted trie metadata.
|
||||
*
|
||||
* @param dataOutput output stream
|
||||
* @param metadata metadata to serialize
|
||||
* @throws IOException if writing fails
|
||||
*/
|
||||
private static void writeMetadata(final DataOutputStream dataOutput, final TrieMetadata metadata)
|
||||
throws IOException {
|
||||
dataOutput.writeUTF(metadata.toTextBlock());
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads persisted trie metadata while remaining backward compatible with
|
||||
* earlier stream versions.
|
||||
*
|
||||
* @param dataInput input stream
|
||||
* @param version persisted stream version
|
||||
* @return deserialized metadata
|
||||
* @throws IOException if the metadata section is invalid
|
||||
*/
|
||||
private static TrieMetadata readMetadata(final DataInputStream dataInput, final int version) throws IOException {
|
||||
if (version >= 5) { // NOPMD
|
||||
try {
|
||||
return TrieMetadata.fromTextBlock(version, dataInput.readUTF());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IOException("Invalid metadata block.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
final WordTraversalDirection traversalDirection;
|
||||
if (version >= 2) { // NOPMD
|
||||
final int traversalDirectionOrdinal = dataInput.readInt();
|
||||
final WordTraversalDirection[] traversalDirections = WordTraversalDirection.values();
|
||||
if (traversalDirectionOrdinal < 0 || traversalDirectionOrdinal >= traversalDirections.length) {
|
||||
throw new IOException("Invalid traversal direction ordinal: " + traversalDirectionOrdinal);
|
||||
}
|
||||
traversalDirection = traversalDirections[traversalDirectionOrdinal];
|
||||
} else {
|
||||
traversalDirection = WordTraversalDirection.BACKWARD;
|
||||
}
|
||||
|
||||
if (version < 3) { // NOPMD
|
||||
return TrieMetadata.legacy(version, traversalDirection);
|
||||
}
|
||||
|
||||
final ReductionMode[] reductionModes = ReductionMode.values();
|
||||
final int reductionModeOrdinal = dataInput.readInt();
|
||||
if (reductionModeOrdinal < 0 || reductionModeOrdinal >= reductionModes.length) {
|
||||
throw new IOException("Invalid reduction mode ordinal: " + reductionModeOrdinal);
|
||||
}
|
||||
|
||||
final int dominantWinnerMinPercent = dataInput.readInt();
|
||||
final int dominantWinnerOverSecondRatio = dataInput.readInt(); // NOPMD
|
||||
|
||||
final DiacriticProcessingMode[] diacriticProcessingModes = DiacriticProcessingMode.values();
|
||||
final int diacriticProcessingModeOrdinal = dataInput.readInt(); // NOPMD
|
||||
if (diacriticProcessingModeOrdinal < 0 || diacriticProcessingModeOrdinal >= diacriticProcessingModes.length) {
|
||||
throw new IOException("Invalid diacritic processing mode ordinal: " + diacriticProcessingModeOrdinal);
|
||||
}
|
||||
|
||||
final CaseProcessingMode caseProcessingMode;
|
||||
if (version >= 4) { // NOPMD
|
||||
final CaseProcessingMode[] caseProcessingModes = CaseProcessingMode.values();
|
||||
final int caseProcessingModeOrdinal = dataInput.readInt();
|
||||
if (caseProcessingModeOrdinal < 0 || caseProcessingModeOrdinal >= caseProcessingModes.length) {
|
||||
throw new IOException("Invalid case processing mode ordinal: " + caseProcessingModeOrdinal);
|
||||
}
|
||||
caseProcessingMode = caseProcessingModes[caseProcessingModeOrdinal];
|
||||
} else {
|
||||
caseProcessingMode = CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||
}
|
||||
|
||||
return new TrieMetadata(version, traversalDirection,
|
||||
new ReductionSettings(reductionModes[reductionModeOrdinal], dominantWinnerMinPercent,
|
||||
dominantWinnerOverSecondRatio),
|
||||
diacriticProcessingModes[diacriticProcessingModeOrdinal], caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -501,13 +639,14 @@ public final class FrequencyTrie<V> {
|
||||
/**
|
||||
* Locates the compiled node for the supplied key.
|
||||
*
|
||||
* @param key key to resolve
|
||||
* @param key already-normalized key to resolve
|
||||
* @return compiled node, or {@code null} if the path does not exist
|
||||
*/
|
||||
private CompiledNode<V> findNode(final String key) {
|
||||
CompiledNode<V> current = this.root;
|
||||
for (int index = 0; index < key.length(); index++) {
|
||||
current = current.findChild(key.charAt(index));
|
||||
for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) {
|
||||
current = current.findChild(
|
||||
key.charAt(this.metadata.traversalDirection().logicalIndex(key.length(), traversalOffset)));
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
@@ -515,6 +654,29 @@ public final class FrequencyTrie<V> {
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies lookup-time case normalization according to persisted metadata.
|
||||
*
|
||||
* @param key lookup key
|
||||
* @return normalized key for trie traversal
|
||||
*/
|
||||
private String normalizeLookupKey(final String key) {
|
||||
String normalized = key;
|
||||
|
||||
if (this.metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT) {
|
||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
if (this.metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE) {
|
||||
normalized = DiacriticStripper.strip(normalized);
|
||||
} else if (this.metadata.diacriticProcessingMode() == DiacriticProcessingMode.AS_IS_AND_STRIPPED_FALLBACK) {
|
||||
throw new UnsupportedOperationException(
|
||||
"Diacritic processing mode AS_IS_AND_STRIPPED_FALLBACK is not supported yet.");
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builder of {@link FrequencyTrie}.
|
||||
*
|
||||
@@ -544,6 +706,21 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private final ReductionSettings reductionSettings;
|
||||
|
||||
/**
|
||||
* Logical key traversal direction used by this builder.
|
||||
*/
|
||||
private final WordTraversalDirection traversalDirection;
|
||||
|
||||
/**
|
||||
* Dictionary case processing mode associated with this builder.
|
||||
*/
|
||||
private final CaseProcessingMode caseProcessingMode;
|
||||
|
||||
/**
|
||||
* Dictionary diacritic processing mode associated with this builder.
|
||||
*/
|
||||
private final DiacriticProcessingMode diacriticProcessingMode;
|
||||
|
||||
/**
|
||||
* Mutable root node.
|
||||
*/
|
||||
@@ -552,13 +729,69 @@ public final class FrequencyTrie<V> {
|
||||
/**
|
||||
* Creates a new builder with the provided settings.
|
||||
*
|
||||
* <p>
|
||||
* This constructor preserves the historical Egothor behavior and therefore
|
||||
* traverses logical keys from their end toward their beginning.
|
||||
* </p>
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings) {
|
||||
this(arrayFactory, reductionSettings, WordTraversalDirection.BACKWARD);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder with the provided settings and explicit traversal
|
||||
* direction.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
this(arrayFactory, reductionSettings, traversalDirection, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder with the provided settings, explicit traversal
|
||||
* direction, and explicit case processing mode.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param caseProcessingMode dictionary case processing mode
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) {
|
||||
this(arrayFactory, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder with the provided settings, explicit traversal
|
||||
* direction, explicit case processing mode, and explicit diacritic processing
|
||||
* mode.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param caseProcessingMode dictionary case processing mode
|
||||
* @param diacriticProcessingMode dictionary diacritic processing mode
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode,
|
||||
final DiacriticProcessingMode diacriticProcessingMode) {
|
||||
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
this.reductionSettings = Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
this.caseProcessingMode = Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
this.diacriticProcessingMode = Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||
this.root = new MutableNode<>();
|
||||
}
|
||||
|
||||
@@ -566,12 +799,31 @@ public final class FrequencyTrie<V> {
|
||||
* Creates a new builder using default thresholds for the supplied reduction
|
||||
* mode.
|
||||
*
|
||||
* <p>
|
||||
* This constructor preserves the historical Egothor behavior and therefore
|
||||
* traverses logical keys from their end toward their beginning.
|
||||
* </p>
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionMode reduction mode
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionMode reductionMode) {
|
||||
this(arrayFactory, ReductionSettings.withDefaults(reductionMode));
|
||||
this(arrayFactory, ReductionSettings.withDefaults(reductionMode), WordTraversalDirection.BACKWARD);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder using default thresholds for the supplied reduction
|
||||
* mode and explicit traversal direction.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionMode reduction mode
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionMode reductionMode,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
this(arrayFactory, ReductionSettings.withDefaults(reductionMode), traversalDirection);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -611,7 +863,9 @@ public final class FrequencyTrie<V> {
|
||||
reductionContext.canonicalNodeCount());
|
||||
}
|
||||
|
||||
return new FrequencyTrie<>(this.arrayFactory, compiledRoot);
|
||||
final TrieMetadata metadata = TrieMetadata.forCompilation(this.traversalDirection, this.reductionSettings,
|
||||
this.diacriticProcessingMode, this.caseProcessingMode);
|
||||
return new FrequencyTrie<>(this.arrayFactory, compiledRoot, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -645,9 +899,12 @@ public final class FrequencyTrie<V> {
|
||||
throw new IllegalArgumentException("count must be at least 1.");
|
||||
}
|
||||
|
||||
final String normalizedKey = normalizeDictionaryKey(key);
|
||||
|
||||
MutableNode<V> current = this.root;
|
||||
for (int index = 0; index < key.length(); index++) {
|
||||
final Character edge = key.charAt(index);
|
||||
for (int traversalOffset = 0; traversalOffset < normalizedKey.length(); traversalOffset++) {
|
||||
final Character edge = normalizedKey
|
||||
.charAt(this.traversalDirection.logicalIndex(normalizedKey.length(), traversalOffset));
|
||||
MutableNode<V> child = current.children().get(edge);
|
||||
if (child == null) {
|
||||
child = new MutableNode<>(); // NOPMD
|
||||
@@ -665,6 +922,30 @@ public final class FrequencyTrie<V> {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies build-time dictionary-key normalization according to the builder
|
||||
* configuration.
|
||||
*
|
||||
* @param key dictionary key
|
||||
* @return normalized key for trie insertion
|
||||
*/
|
||||
private String normalizeDictionaryKey(final String key) {
|
||||
String normalized = key;
|
||||
|
||||
if (this.caseProcessingMode == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT) {
|
||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
if (this.diacriticProcessingMode == DiacriticProcessingMode.REMOVE) {
|
||||
normalized = DiacriticStripper.strip(normalized);
|
||||
} else if (this.diacriticProcessingMode == DiacriticProcessingMode.AS_IS_AND_STRIPPED_FALLBACK) {
|
||||
throw new UnsupportedOperationException(
|
||||
"Diacritic processing mode AS_IS_AND_STRIPPED_FALLBACK is not supported yet.");
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of mutable build-time nodes currently reachable from the
|
||||
* builder root.
|
||||
@@ -679,6 +960,15 @@ public final class FrequencyTrie<V> {
|
||||
return countMutableNodes(this.root);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the logical key traversal direction used by this builder.
|
||||
*
|
||||
* @return logical key traversal direction
|
||||
*/
|
||||
/* default */ WordTraversalDirection traversalDirection() {
|
||||
return this.traversalDirection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts mutable nodes recursively.
|
||||
*
|
||||
|
||||
@@ -87,10 +87,12 @@ public final class FrequencyTrieBuilders {
|
||||
Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
|
||||
final FrequencyTrie.Builder<V> builder = new FrequencyTrie.Builder<>(arrayFactory, reductionSettings);
|
||||
final FrequencyTrie.Builder<V> builder = new FrequencyTrie.Builder<>(arrayFactory, reductionSettings,
|
||||
source.traversalDirection(), source.metadata().caseProcessingMode(),
|
||||
source.metadata().diacriticProcessingMode());
|
||||
final StringBuilder keyBuilder = new StringBuilder(64);
|
||||
|
||||
copyNode(source.root(), keyBuilder, builder);
|
||||
copyNode(source.root(), keyBuilder, builder, source.traversalDirection());
|
||||
|
||||
LOGGER.log(Level.FINE, "Reconstructed writable builder from compiled trie.");
|
||||
return builder;
|
||||
@@ -119,18 +121,20 @@ public final class FrequencyTrieBuilders {
|
||||
*
|
||||
* @param node current compiled node
|
||||
* @param keyBuilder current key builder
|
||||
* @param builder target mutable builder
|
||||
* @param <V> value type
|
||||
* @param builder target mutable builder
|
||||
* @param traversalDirection logical key traversal direction used by the source
|
||||
* @param <V> value type
|
||||
*/
|
||||
private static <V> void copyNode(final CompiledNode<V> node, final StringBuilder keyBuilder,
|
||||
final FrequencyTrie.Builder<V> builder) {
|
||||
final FrequencyTrie.Builder<V> builder, final WordTraversalDirection traversalDirection) {
|
||||
final String logicalKey = traversalDirection.traversalPathToLogicalKey(keyBuilder);
|
||||
for (int valueIndex = 0; valueIndex < node.orderedValues().length; valueIndex++) {
|
||||
builder.put(keyBuilder.toString(), node.orderedValues()[valueIndex], node.orderedCounts()[valueIndex]);
|
||||
builder.put(logicalKey, node.orderedValues()[valueIndex], node.orderedCounts()[valueIndex]);
|
||||
}
|
||||
|
||||
for (int childIndex = 0; childIndex < node.edgeLabels().length; childIndex++) {
|
||||
keyBuilder.append(node.edgeLabels()[childIndex]);
|
||||
copyNode(node.children()[childIndex], keyBuilder, builder);
|
||||
copyNode(node.children()[childIndex], keyBuilder, builder, traversalDirection);
|
||||
keyBuilder.setLength(keyBuilder.length() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
@@ -37,10 +38,19 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
* and applies such commands back to source words.
|
||||
*
|
||||
* <p>
|
||||
* The generated patch command follows the historical Egothor convention:
|
||||
* instructions are serialized so that they are applied from the end of the
|
||||
* source word toward its beginning. This keeps the command stream compact and
|
||||
* matches the behavior expected by existing stemming data.
|
||||
* The historical Egothor patch language is defined for backward traversal, that
|
||||
* is, from the logical end of a word toward its beginning. This implementation
|
||||
* preserves that proven opcode semantics as the single internal representation.
|
||||
* Forward traversal is implemented by translating source and target words to
|
||||
* the equivalent reversed logical form at the API boundary and then delegating
|
||||
* to the same backward encoder and decoder.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* This design keeps the patch language stable, avoids maintaining two distinct
|
||||
* opcode interpreters, and guarantees that forward traversal is semantically
|
||||
* equivalent to running the historical algorithm on the reversed logical word
|
||||
* form.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
@@ -57,8 +67,19 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
* instance can still be used safely when needed.
|
||||
* </p>
|
||||
*/
|
||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||
public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
* Backward direction apply strategy with no runtime direction branching.
|
||||
*/
|
||||
private static final ApplyStrategy BACKWARD_APPLY_STRATEGY = PatchCommandEncoder::applyBackward;
|
||||
|
||||
/**
|
||||
* Forward direction apply strategy with no runtime direction branching.
|
||||
*/
|
||||
private static final ApplyStrategy FORWARD_APPLY_STRATEGY = PatchCommandEncoder::applyForward;
|
||||
|
||||
/**
|
||||
* Serialized opcode for deleting one or more characters.
|
||||
*/
|
||||
@@ -87,12 +108,6 @@ public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
* Serialized opcode for a canonical no-operation patch.
|
||||
*
|
||||
* <p>
|
||||
* This opcode represents an identity transform of the whole source word. It is
|
||||
* used to ensure that equal source and target words always produce the same
|
||||
* serialized patch command.
|
||||
* </p>
|
||||
*/
|
||||
private static final char NOOP_OPCODE = 'N';
|
||||
|
||||
@@ -103,11 +118,6 @@ public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
* Canonical serialized no-operation patch.
|
||||
*
|
||||
* <p>
|
||||
* This constant is returned by {@link #encode(String, String)} whenever source
|
||||
* and target are equal.
|
||||
* </p>
|
||||
*/
|
||||
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
|
||||
|
||||
@@ -118,13 +128,6 @@ public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
* Extra matrix headroom reserved beyond the immediately required dimensions.
|
||||
*
|
||||
* <p>
|
||||
* A small fixed margin reduces repeated reallocation when a caller encodes many
|
||||
* similarly sized terms in sequence. The value is intentionally modest: large
|
||||
* enough to absorb minor size fluctuations, yet small enough to avoid
|
||||
* materially over-allocating the reused dynamic-programming matrices.
|
||||
* </p>
|
||||
*/
|
||||
private static final int CAPACITY_MARGIN = 8;
|
||||
|
||||
@@ -148,6 +151,17 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
private final int matchCost;
|
||||
|
||||
/**
|
||||
* Direction in which words are traversed during both patch serialization and
|
||||
* patch application.
|
||||
*/
|
||||
private final WordTraversalDirection traversalDirection;
|
||||
|
||||
/**
|
||||
* Direction-specialized patch apply strategy.
|
||||
*/
|
||||
private final ApplyStrategy applyStrategy;
|
||||
|
||||
/**
|
||||
* Currently allocated source dimension of reusable matrices.
|
||||
*/
|
||||
@@ -178,53 +192,49 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
private enum Trace {
|
||||
|
||||
/**
|
||||
* Deletes one character from the source sequence.
|
||||
*/
|
||||
/** Deletes one character from the source sequence. */
|
||||
DELETE,
|
||||
|
||||
/**
|
||||
* Inserts one character from the target sequence.
|
||||
*/
|
||||
/** Inserts one character from the target sequence. */
|
||||
INSERT,
|
||||
|
||||
/**
|
||||
* Replaces one source character with one target character.
|
||||
*/
|
||||
/** Replaces one source character with one target character. */
|
||||
REPLACE,
|
||||
|
||||
/**
|
||||
* Keeps one matching character unchanged.
|
||||
*/
|
||||
/** Keeps one matching character unchanged. */
|
||||
MATCH
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with the traditional Egothor cost model: insert = 1,
|
||||
* delete = 1, replace = 1, match = 0.
|
||||
* Direction-specialized patch application strategy.
|
||||
*/
|
||||
public PatchCommandEncoder() {
|
||||
this(1, 1, 1, 0);
|
||||
@FunctionalInterface
|
||||
private interface ApplyStrategy {
|
||||
/**
|
||||
* Applies the command.
|
||||
*
|
||||
* @param source original text
|
||||
* @param patchCommand patch command
|
||||
* @return final text after applying the command
|
||||
*/
|
||||
String apply(String source, String patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with explicit operation costs.
|
||||
*
|
||||
* @param insertCost cost of inserting one character
|
||||
* @param deleteCost cost of deleting one character
|
||||
* @param replaceCost cost of replacing one character
|
||||
* @param matchCost cost of keeping one equal character unchanged
|
||||
*/
|
||||
public PatchCommandEncoder(int insertCost, int deleteCost, int replaceCost, int matchCost) {
|
||||
private PatchCommandEncoder(final Builder builder) {
|
||||
this.traversalDirection = Objects.requireNonNull(builder.traversalDirection, "traversalDirection");
|
||||
final int insertCost = builder.insertCost;
|
||||
if (insertCost < 0) {
|
||||
throw new IllegalArgumentException("insertCost must be non-negative.");
|
||||
}
|
||||
final int deleteCost = builder.deleteCost;
|
||||
if (deleteCost < 0) {
|
||||
throw new IllegalArgumentException("deleteCost must be non-negative.");
|
||||
}
|
||||
final int replaceCost = builder.replaceCost;
|
||||
if (replaceCost < 0) {
|
||||
throw new IllegalArgumentException("replaceCost must be non-negative.");
|
||||
}
|
||||
final int matchCost = builder.matchCost;
|
||||
if (matchCost < 0) {
|
||||
throw new IllegalArgumentException("matchCost must be non-negative.");
|
||||
}
|
||||
@@ -233,12 +243,22 @@ public final class PatchCommandEncoder {
|
||||
this.deleteCost = deleteCost;
|
||||
this.replaceCost = replaceCost;
|
||||
this.matchCost = matchCost;
|
||||
this.applyStrategy = applyStrategyFor(this.traversalDirection);
|
||||
this.sourceCapacity = 0;
|
||||
this.targetCapacity = 0;
|
||||
this.costMatrix = new int[0][0];
|
||||
this.traceMatrix = new Trace[0][0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a fluent builder for constructing a direction-specialized encoder.
|
||||
*
|
||||
* @return new builder instance
|
||||
*/
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces a compact patch command that transforms {@code source} into
|
||||
* {@code target}.
|
||||
@@ -248,55 +268,139 @@ public final class PatchCommandEncoder {
|
||||
* @return compact patch command, or {@code null} when any argument is
|
||||
* {@code null}
|
||||
*/
|
||||
public String encode(String source, String target) {
|
||||
public String encode(final String source, final String target) {
|
||||
if (source == null || target == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (source.equals(target)) {
|
||||
return NOOP_PATCH;
|
||||
}
|
||||
|
||||
int sourceLength = source.length();
|
||||
int targetLength = target.length();
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
ensureCapacity(sourceLength + 1, targetLength + 1);
|
||||
initializeBoundaryConditions(sourceLength, targetLength);
|
||||
|
||||
char[] sourceCharacters = source.toCharArray();
|
||||
char[] targetCharacters = target.toCharArray();
|
||||
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength);
|
||||
|
||||
return buildPatchCommand(targetCharacters, sourceLength, targetLength);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
if (this.traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
return encodeBackward(source, target);
|
||||
}
|
||||
return encodeForward(source, target);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command to the supplied source word.
|
||||
* Applies a compact patch command using this encoder instance traversal
|
||||
* direction.
|
||||
*
|
||||
* <p>
|
||||
* This method operates directly on serialized opcodes rather than mapping them
|
||||
* to another representation. That keeps the hot path small and avoids
|
||||
* unnecessary indirection during patch application.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* For compatibility with the historical behavior, malformed patch input that
|
||||
* causes index failures results in the original source word being returned
|
||||
* unchanged.
|
||||
* This is the branch-free instance-level fast path for repeated patch
|
||||
* application in a known traversal direction.
|
||||
* </p>
|
||||
*
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
public String applyWithConfiguredDirection(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
return this.applyStrategy.apply(source, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command to the supplied source word using the
|
||||
* historical backward traversal direction.
|
||||
*
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
public static String apply(final String source, final String patchCommand) {
|
||||
return apply(source, patchCommand, WordTraversalDirection.BACKWARD);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command to the supplied source word using the
|
||||
* specified traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* The implementation uses dedicated direction-specific patch decoders.
|
||||
* </p>
|
||||
*
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @param traversalDirection traversal direction used by the patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
public static String apply(final String source, final String patchCommand,
|
||||
final WordTraversalDirection traversalDirection) {
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a patch command using the historical backward Egothor semantics.
|
||||
*
|
||||
* @param source source word form in legacy backward logical space
|
||||
* @param target target word form in legacy backward logical space
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String encodeBackward(final String source, final String target) {
|
||||
final int sourceLength = source.length();
|
||||
final int targetLength = target.length();
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
ensureCapacity(sourceLength + 1, targetLength + 1);
|
||||
initializeBoundaryConditionsBackward(sourceLength, targetLength);
|
||||
|
||||
final char[] sourceCharacters = source.toCharArray();
|
||||
final char[] targetCharacters = target.toCharArray();
|
||||
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength,
|
||||
WordTraversalDirection.BACKWARD);
|
||||
|
||||
return buildPatchCommandBackward(targetCharacters, sourceLength, targetLength);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a patch command using forward traversal semantics.
|
||||
*
|
||||
* @param source source word form
|
||||
* @param target target word form
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String encodeForward(final String source, final String target) {
|
||||
final int sourceLength = source.length();
|
||||
final int targetLength = target.length();
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
ensureCapacity(sourceLength + 1, targetLength + 1);
|
||||
initializeBoundaryConditionsForward(sourceLength, targetLength);
|
||||
|
||||
final char[] sourceCharacters = source.toCharArray();
|
||||
final char[] targetCharacters = target.toCharArray();
|
||||
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength,
|
||||
WordTraversalDirection.FORWARD);
|
||||
|
||||
return buildPatchCommandForward(targetCharacters, sourceLength, targetLength);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a patch command using the historical backward Egothor semantics.
|
||||
*
|
||||
* @param source original source word in legacy backward logical space
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
||||
public static String apply(String source, String patchCommand) {
|
||||
private static String applyBackward(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
@@ -306,24 +410,21 @@ public final class PatchCommandEncoder {
|
||||
if (NOOP_PATCH.equals(patchCommand)) {
|
||||
return source;
|
||||
}
|
||||
|
||||
if ((patchCommand.length() & 1) != 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
StringBuilder result = new StringBuilder(source);
|
||||
|
||||
final StringBuilder result = new StringBuilder(source);
|
||||
if (result.isEmpty()) {
|
||||
return applyToEmptySource(result, patchCommand);
|
||||
return applyBackwardToEmptySource(result, patchCommand);
|
||||
}
|
||||
|
||||
int position = result.length() - 1;
|
||||
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
|
||||
char opcode = patchCommand.charAt(patchIndex);
|
||||
char argument = patchCommand.charAt(patchIndex + 1);
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
@@ -343,7 +444,7 @@ public final class PatchCommandEncoder {
|
||||
if (deleteCount < 1) {
|
||||
return source;
|
||||
}
|
||||
int deleteEndExclusive = position + 1;
|
||||
final int deleteEndExclusive = position + 1;
|
||||
position -= deleteCount - 1;
|
||||
result.delete(position, deleteEndExclusive);
|
||||
break;
|
||||
@@ -373,27 +474,86 @@ public final class PatchCommandEncoder {
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a compact count argument used by skip and delete instructions.
|
||||
* Applies a patch command using forward traversal semantics.
|
||||
*
|
||||
* <p>
|
||||
* Valid encoded counts start at {@code 'a'} for one affected character. Values
|
||||
* below {@code 'a'} are malformed and are reported to callers via the
|
||||
* compatibility fallback path rather than by throwing a dedicated exception.
|
||||
* </p>
|
||||
*
|
||||
* @param argument serialized count argument
|
||||
* @return decoded positive count, or {@code -1} when the argument is malformed
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
private static int decodeEncodedCount(final char argument) {
|
||||
if (argument < 'a') {
|
||||
return -1;
|
||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
||||
private static String applyForward(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
return argument - 'a' + 1;
|
||||
if (patchCommand == null || patchCommand.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
if (NOOP_PATCH.equals(patchCommand)) {
|
||||
return source;
|
||||
}
|
||||
if ((patchCommand.length() & 1) != 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
final StringBuilder result = new StringBuilder(source);
|
||||
if (result.isEmpty()) {
|
||||
return applyForwardToEmptySource(result, patchCommand);
|
||||
}
|
||||
|
||||
int position = 0;
|
||||
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return source;
|
||||
}
|
||||
position = position + skipCount - 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
result.setCharAt(position, argument);
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1) {
|
||||
return source;
|
||||
}
|
||||
result.delete(position, position + deleteCount);
|
||||
position--;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
result.insert(position, argument);
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
|
||||
}
|
||||
return source;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
|
||||
}
|
||||
|
||||
position++;
|
||||
}
|
||||
} catch (IndexOutOfBoundsException exception) {
|
||||
return source;
|
||||
}
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a patch command to an empty source word.
|
||||
* Applies a backward patch command to an empty source word.
|
||||
*
|
||||
* <p>
|
||||
* Only insertion instructions are meaningful for an empty source. Skip,
|
||||
@@ -407,12 +567,11 @@ public final class PatchCommandEncoder {
|
||||
* @return transformed word, or the original empty word when the patch is
|
||||
* malformed
|
||||
*/
|
||||
private static String applyToEmptySource(StringBuilder result, String patchCommand) {
|
||||
private static String applyBackwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
|
||||
char opcode = patchCommand.charAt(patchIndex);
|
||||
char argument = patchCommand.charAt(patchIndex + 1);
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case INSERT_OPCODE:
|
||||
@@ -441,6 +600,71 @@ public final class PatchCommandEncoder {
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a forward patch command to an empty source word.
|
||||
*
|
||||
* @param result empty result builder
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or the original empty word when the patch is
|
||||
* malformed
|
||||
*/
|
||||
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case INSERT_OPCODE:
|
||||
result.append(argument);
|
||||
break;
|
||||
|
||||
case SKIP_OPCODE:
|
||||
case REPLACE_OPCODE:
|
||||
case DELETE_OPCODE:
|
||||
return "";
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
|
||||
}
|
||||
return "";
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
|
||||
}
|
||||
}
|
||||
} catch (IndexOutOfBoundsException exception) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the direction-specialized apply strategy.
|
||||
*
|
||||
* @param traversalDirection requested traversal direction
|
||||
* @return branch-free apply strategy for that direction
|
||||
*/
|
||||
private static ApplyStrategy applyStrategyFor(final WordTraversalDirection traversalDirection) {
|
||||
return traversalDirection == WordTraversalDirection.BACKWARD ? BACKWARD_APPLY_STRATEGY : FORWARD_APPLY_STRATEGY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a compact count argument used by skip and delete instructions.
|
||||
*
|
||||
* @param argument serialized count argument
|
||||
* @return decoded positive count, or {@code -1} when the argument is malformed
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
private static int decodeEncodedCount(final char argument) {
|
||||
if (argument < 'a') {
|
||||
return -1;
|
||||
}
|
||||
return argument - 'a' + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that internal matrices are large enough for the requested input
|
||||
* dimensions.
|
||||
@@ -448,16 +672,16 @@ public final class PatchCommandEncoder {
|
||||
* @param requiredSourceCapacity required source dimension
|
||||
* @param requiredTargetCapacity required target dimension
|
||||
*/
|
||||
private void ensureCapacity(int requiredSourceCapacity, int requiredTargetCapacity) {
|
||||
if (requiredSourceCapacity <= sourceCapacity && requiredTargetCapacity <= targetCapacity) {
|
||||
private void ensureCapacity(final int requiredSourceCapacity, final int requiredTargetCapacity) {
|
||||
if (requiredSourceCapacity <= this.sourceCapacity && requiredTargetCapacity <= this.targetCapacity) {
|
||||
return;
|
||||
}
|
||||
|
||||
sourceCapacity = Math.max(sourceCapacity, requiredSourceCapacity) + CAPACITY_MARGIN;
|
||||
targetCapacity = Math.max(targetCapacity, requiredTargetCapacity) + CAPACITY_MARGIN;
|
||||
this.sourceCapacity = Math.max(this.sourceCapacity, requiredSourceCapacity) + CAPACITY_MARGIN;
|
||||
this.targetCapacity = Math.max(this.targetCapacity, requiredTargetCapacity) + CAPACITY_MARGIN;
|
||||
|
||||
costMatrix = new int[sourceCapacity][targetCapacity];
|
||||
traceMatrix = new Trace[sourceCapacity][targetCapacity];
|
||||
this.costMatrix = new int[this.sourceCapacity][this.targetCapacity];
|
||||
this.traceMatrix = new Trace[this.sourceCapacity][this.targetCapacity];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -467,18 +691,41 @@ public final class PatchCommandEncoder {
|
||||
* @param sourceLength length of the source word
|
||||
* @param targetLength length of the target word
|
||||
*/
|
||||
private void initializeBoundaryConditions(int sourceLength, int targetLength) {
|
||||
costMatrix[0][0] = 0;
|
||||
traceMatrix[0][0] = Trace.MATCH;
|
||||
private void initializeBoundaryConditionsBackward(final int sourceLength, final int targetLength) {
|
||||
this.costMatrix[0][0] = 0;
|
||||
this.traceMatrix[0][0] = Trace.MATCH;
|
||||
|
||||
for (int sourceIndex = 1; sourceIndex <= sourceLength; sourceIndex++) {
|
||||
costMatrix[sourceIndex][0] = sourceIndex * deleteCost;
|
||||
traceMatrix[sourceIndex][0] = Trace.DELETE;
|
||||
this.costMatrix[sourceIndex][0] = sourceIndex * this.deleteCost;
|
||||
this.traceMatrix[sourceIndex][0] = Trace.DELETE;
|
||||
}
|
||||
|
||||
for (int targetIndex = 1; targetIndex <= targetLength; targetIndex++) {
|
||||
costMatrix[0][targetIndex] = targetIndex * insertCost;
|
||||
traceMatrix[0][targetIndex] = Trace.INSERT;
|
||||
this.costMatrix[0][targetIndex] = targetIndex * this.insertCost;
|
||||
this.traceMatrix[0][targetIndex] = Trace.INSERT;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes boundary conditions for forward dynamic-programming traversal.
|
||||
*
|
||||
* @param sourceLength length of the source word
|
||||
* @param targetLength length of the target word
|
||||
*/
|
||||
private void initializeBoundaryConditionsForward(final int sourceLength, final int targetLength) {
|
||||
this.costMatrix[sourceLength][targetLength] = 0;
|
||||
this.traceMatrix[sourceLength][targetLength] = Trace.MATCH;
|
||||
|
||||
for (int sourceIndex = sourceLength - 1; sourceIndex >= 0; sourceIndex--) {
|
||||
this.costMatrix[sourceIndex][targetLength] = this.costMatrix[sourceIndex + 1][targetLength]
|
||||
+ this.deleteCost;
|
||||
this.traceMatrix[sourceIndex][targetLength] = Trace.DELETE;
|
||||
}
|
||||
|
||||
for (int targetIndex = targetLength - 1; targetIndex >= 0; targetIndex--) {
|
||||
this.costMatrix[sourceLength][targetIndex] = this.costMatrix[sourceLength][targetIndex + 1]
|
||||
+ this.insertCost;
|
||||
this.traceMatrix[sourceLength][targetIndex] = Trace.INSERT;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -490,20 +737,58 @@ public final class PatchCommandEncoder {
|
||||
* @param targetCharacters target characters
|
||||
* @param sourceLength source length
|
||||
* @param targetLength target length
|
||||
* @param direction traversal direction used to compare characters
|
||||
*/
|
||||
private void fillMatrices(char[] sourceCharacters, char[] targetCharacters, int sourceLength, int targetLength) {
|
||||
private void fillMatrices(final char[] sourceCharacters, final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength, final WordTraversalDirection direction) {
|
||||
final int sourceStart;
|
||||
final int sourceEndExclusive;
|
||||
final int sourceStep;
|
||||
final int targetStart;
|
||||
final int targetEndExclusive;
|
||||
final int targetStep;
|
||||
final int sourceCharacterOffset;
|
||||
final int targetCharacterOffset;
|
||||
final int sourceNeighborDelta;
|
||||
final int targetNeighborDelta;
|
||||
|
||||
for (int sourceIndex = 1; sourceIndex <= sourceLength; sourceIndex++) {
|
||||
char sourceCharacter = sourceCharacters[sourceIndex - 1];
|
||||
if (direction == WordTraversalDirection.BACKWARD) {
|
||||
sourceStart = 1;
|
||||
sourceEndExclusive = sourceLength + 1;
|
||||
sourceStep = 1;
|
||||
targetStart = 1;
|
||||
targetEndExclusive = targetLength + 1;
|
||||
targetStep = 1;
|
||||
sourceCharacterOffset = -1;
|
||||
targetCharacterOffset = -1;
|
||||
sourceNeighborDelta = -1;
|
||||
targetNeighborDelta = -1;
|
||||
} else {
|
||||
sourceStart = sourceLength - 1;
|
||||
sourceEndExclusive = -1;
|
||||
sourceStep = -1;
|
||||
targetStart = targetLength - 1;
|
||||
targetEndExclusive = -1;
|
||||
targetStep = -1;
|
||||
sourceCharacterOffset = 0;
|
||||
targetCharacterOffset = 0;
|
||||
sourceNeighborDelta = 1;
|
||||
targetNeighborDelta = 1;
|
||||
}
|
||||
|
||||
for (int targetIndex = 1; targetIndex <= targetLength; targetIndex++) {
|
||||
char targetCharacter = targetCharacters[targetIndex - 1];
|
||||
for (int sourceIndex = sourceStart; sourceIndex != sourceEndExclusive; sourceIndex += sourceStep) {
|
||||
final char sourceCharacter = sourceCharacters[sourceIndex + sourceCharacterOffset];
|
||||
final int sourceNeighbor = sourceIndex + sourceNeighborDelta;
|
||||
|
||||
int deleteCandidate = costMatrix[sourceIndex - 1][targetIndex] + deleteCost;
|
||||
int insertCandidate = costMatrix[sourceIndex][targetIndex - 1] + insertCost;
|
||||
int replaceCandidate = costMatrix[sourceIndex - 1][targetIndex - 1] + replaceCost;
|
||||
int matchCandidate = costMatrix[sourceIndex - 1][targetIndex - 1]
|
||||
+ (sourceCharacter == targetCharacter ? matchCost : MISMATCH_PENALTY);
|
||||
for (int targetIndex = targetStart; targetIndex != targetEndExclusive; targetIndex += targetStep) {
|
||||
final char targetCharacter = targetCharacters[targetIndex + targetCharacterOffset];
|
||||
final int targetNeighbor = targetIndex + targetNeighborDelta;
|
||||
|
||||
final int deleteCandidate = this.costMatrix[sourceNeighbor][targetIndex] + this.deleteCost;
|
||||
final int insertCandidate = this.costMatrix[sourceIndex][targetNeighbor] + this.insertCost;
|
||||
final int replaceCandidate = this.costMatrix[sourceNeighbor][targetNeighbor] + this.replaceCost;
|
||||
final int matchCandidate = this.costMatrix[sourceNeighbor][targetNeighbor]
|
||||
+ (sourceCharacter == targetCharacter ? this.matchCost : MISMATCH_PENALTY);
|
||||
|
||||
int bestCost = matchCandidate;
|
||||
Trace bestTrace = Trace.MATCH;
|
||||
@@ -521,8 +806,8 @@ public final class PatchCommandEncoder {
|
||||
bestTrace = Trace.REPLACE;
|
||||
}
|
||||
|
||||
costMatrix[sourceIndex][targetIndex] = bestCost;
|
||||
traceMatrix[sourceIndex][targetIndex] = bestTrace;
|
||||
this.costMatrix[sourceIndex][targetIndex] = bestCost;
|
||||
this.traceMatrix[sourceIndex][targetIndex] = bestTrace;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -536,9 +821,9 @@ public final class PatchCommandEncoder {
|
||||
* @param targetLength target length
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String buildPatchCommand(char[] targetCharacters, int sourceLength, int targetLength) {
|
||||
|
||||
StringBuilder patchBuilder = new StringBuilder(sourceLength + targetLength);
|
||||
private String buildPatchCommandBackward(final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength) {
|
||||
final StringBuilder patchBuilder = new StringBuilder(sourceLength + targetLength);
|
||||
|
||||
char pendingDeletes = COUNT_SENTINEL;
|
||||
char pendingSkips = COUNT_SENTINEL;
|
||||
@@ -547,7 +832,7 @@ public final class PatchCommandEncoder {
|
||||
int targetIndex = targetLength;
|
||||
|
||||
while (sourceIndex != 0 || targetIndex != 0) {
|
||||
Trace trace = traceMatrix[sourceIndex][targetIndex];
|
||||
final Trace trace = this.traceMatrix[sourceIndex][targetIndex];
|
||||
|
||||
switch (trace) {
|
||||
case DELETE:
|
||||
@@ -605,6 +890,83 @@ public final class PatchCommandEncoder {
|
||||
return patchBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstructs compact patch command for forward traversal.
|
||||
*
|
||||
* @param targetCharacters target characters
|
||||
* @param sourceLength source length
|
||||
* @param targetLength target length
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String buildPatchCommandForward(final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength) {
|
||||
final StringBuilder patchBuilder = new StringBuilder(sourceLength + targetLength);
|
||||
|
||||
char pendingDeletes = COUNT_SENTINEL;
|
||||
char pendingSkips = COUNT_SENTINEL;
|
||||
|
||||
int sourceIndex = 0;
|
||||
int targetIndex = 0;
|
||||
|
||||
while (sourceIndex != sourceLength || targetIndex != targetLength) {
|
||||
final Trace trace = this.traceMatrix[sourceIndex][targetIndex];
|
||||
|
||||
switch (trace) {
|
||||
case DELETE:
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
pendingDeletes++;
|
||||
sourceIndex++;
|
||||
break;
|
||||
|
||||
case INSERT:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
appendInstruction(patchBuilder, INSERT_OPCODE, targetCharacters[targetIndex]);
|
||||
targetIndex++;
|
||||
break;
|
||||
|
||||
case REPLACE:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
appendInstruction(patchBuilder, REPLACE_OPCODE, targetCharacters[targetIndex]);
|
||||
sourceIndex++;
|
||||
targetIndex++;
|
||||
break;
|
||||
|
||||
case MATCH:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
pendingSkips++;
|
||||
sourceIndex++;
|
||||
targetIndex++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
}
|
||||
|
||||
return patchBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends one serialized instruction to the patch command builder.
|
||||
*
|
||||
@@ -612,7 +974,91 @@ public final class PatchCommandEncoder {
|
||||
* @param opcode single-character instruction opcode
|
||||
* @param argument encoded instruction argument
|
||||
*/
|
||||
private static void appendInstruction(StringBuilder patchBuilder, char opcode, char argument) {
|
||||
private static void appendInstruction(final StringBuilder patchBuilder, final char opcode, final char argument) {
|
||||
patchBuilder.append(opcode).append(argument);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fluent builder for creating direction-specialized {@link PatchCommandEncoder}
|
||||
* instances.
|
||||
*/
|
||||
public static final class Builder {
|
||||
private WordTraversalDirection traversalDirection = WordTraversalDirection.BACKWARD;
|
||||
private int insertCost = 1;
|
||||
private int deleteCost = 1;
|
||||
private int replaceCost = 1;
|
||||
private int matchCost; // = 0
|
||||
|
||||
/**
|
||||
* Creates a builder initialized with the default Egothor-compatible cost model
|
||||
* and backward traversal.
|
||||
*/
|
||||
public Builder() {
|
||||
// Default values are assigned in field initializers.
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets traversal direction used by the created encoder.
|
||||
*
|
||||
* @param value traversal direction
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder traversalDirection(final WordTraversalDirection value) {
|
||||
this.traversalDirection = Objects.requireNonNull(value, "traversalDirection");
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of an insert operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder insertCost(final int value) {
|
||||
this.insertCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a delete operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder deleteCost(final int value) {
|
||||
this.deleteCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a replace operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder replaceCost(final int value) {
|
||||
this.replaceCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a match operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder matchCost(final int value) {
|
||||
this.matchCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a direction-specialized encoder instance.
|
||||
*
|
||||
* @return configured encoder
|
||||
*/
|
||||
public PatchCommandEncoder build() {
|
||||
return new PatchCommandEncoder(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,9 +36,10 @@ import java.io.Reader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.StringTokenizer;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
@@ -46,14 +47,14 @@ import java.util.logging.Logger;
|
||||
* Parser of line-oriented stemmer dictionary files.
|
||||
*
|
||||
* <p>
|
||||
* Each non-empty logical line consists of a stem followed by zero or more known
|
||||
* word variants separated by whitespace. The first token is interpreted as the
|
||||
* canonical stem, and every following token on the same line is interpreted as
|
||||
* a variant belonging to that stem.
|
||||
* Each non-empty logical line uses a tab-separated values layout. The first
|
||||
* column is interpreted as the canonical stem, and every following
|
||||
* tab-separated column on the same line is interpreted as a variant belonging
|
||||
* to that stem.
|
||||
*
|
||||
* <p>
|
||||
* Input lines are normalized to lower case using {@link Locale#ROOT}. Leading
|
||||
* and trailing whitespace is ignored.
|
||||
* Input line case normalization is controlled by {@link CaseProcessingMode}.
|
||||
* Leading and trailing whitespace around each column is ignored.
|
||||
*
|
||||
* <p>
|
||||
* The parser supports line remarks and trailing remarks. The remark markers
|
||||
@@ -61,6 +62,13 @@ import java.util.logging.Logger;
|
||||
* remainder of that line is ignored.
|
||||
*
|
||||
* <p>
|
||||
* Dictionary items containing any Unicode whitespace character are currently
|
||||
* not supported. Such items are ignored and reported through a single
|
||||
* {@link Level#WARNING warning}-level log entry per physical line together with
|
||||
* the source line number, the normalized stem column, and the list of ignored
|
||||
* items from that line.
|
||||
*
|
||||
* <p>
|
||||
* This class is intentionally stateless and allocation-light so it can be used
|
||||
* both by runtime loading and by offline compilation tooling.
|
||||
*/
|
||||
@@ -105,11 +113,27 @@ public final class StemmerDictionaryParser {
|
||||
* @throws IOException if reading fails
|
||||
*/
|
||||
public static ParseStatistics parse(final Path path, final EntryHandler entryHandler) throws IOException {
|
||||
return parse(path, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, entryHandler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a dictionary file from a filesystem path.
|
||||
*
|
||||
* @param path dictionary file path
|
||||
* @param caseProcessingMode case processing mode
|
||||
* @param entryHandler handler receiving parsed entries
|
||||
* @return parsing statistics
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if reading fails
|
||||
*/
|
||||
public static ParseStatistics parse(final Path path, final CaseProcessingMode caseProcessingMode,
|
||||
final EntryHandler entryHandler) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
Objects.requireNonNull(entryHandler, "entryHandler");
|
||||
|
||||
try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
|
||||
return parse(reader, path.toAbsolutePath().toString(), entryHandler);
|
||||
return parse(reader, path.toAbsolutePath().toString(), caseProcessingMode, entryHandler);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,7 +148,23 @@ public final class StemmerDictionaryParser {
|
||||
*/
|
||||
public static ParseStatistics parse(final String fileName, final EntryHandler entryHandler) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
return parse(Path.of(fileName), entryHandler);
|
||||
return parse(Path.of(fileName), CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, entryHandler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a dictionary file from a path string.
|
||||
*
|
||||
* @param fileName dictionary file name or path string
|
||||
* @param caseProcessingMode case processing mode
|
||||
* @param entryHandler handler receiving parsed entries
|
||||
* @return parsing statistics
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if reading fails
|
||||
*/
|
||||
public static ParseStatistics parse(final String fileName, final CaseProcessingMode caseProcessingMode,
|
||||
final EntryHandler entryHandler) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
return parse(Path.of(fileName), caseProcessingMode, entryHandler);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -139,8 +179,25 @@ public final class StemmerDictionaryParser {
|
||||
*/
|
||||
public static ParseStatistics parse(final Reader reader, final String sourceDescription,
|
||||
final EntryHandler entryHandler) throws IOException {
|
||||
return parse(reader, sourceDescription, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, entryHandler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a dictionary from a reader.
|
||||
*
|
||||
* @param reader source reader
|
||||
* @param sourceDescription logical source description for diagnostics
|
||||
* @param caseProcessingMode case processing mode
|
||||
* @param entryHandler handler receiving parsed entries
|
||||
* @return parsing statistics
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if reading or handler processing fails
|
||||
*/
|
||||
public static ParseStatistics parse(final Reader reader, final String sourceDescription,
|
||||
final CaseProcessingMode caseProcessingMode, final EntryHandler entryHandler) throws IOException {
|
||||
Objects.requireNonNull(reader, "reader");
|
||||
Objects.requireNonNull(sourceDescription, "sourceDescription");
|
||||
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
Objects.requireNonNull(entryHandler, "entryHandler");
|
||||
|
||||
final BufferedReader bufferedReader = reader instanceof BufferedReader ? (BufferedReader) reader
|
||||
@@ -153,26 +210,56 @@ public final class StemmerDictionaryParser {
|
||||
for (String line = bufferedReader.readLine(); line != null; line = bufferedReader.readLine()) {
|
||||
lineNumber++;
|
||||
|
||||
final String normalizedLine = stripRemark(line).trim().toLowerCase(Locale.ROOT);
|
||||
final String normalizedLine = normalizeLineCase(stripRemark(line).trim(), caseProcessingMode);
|
||||
if (normalizedLine.isEmpty()) {
|
||||
ignoredLineCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final StringTokenizer tokenizer = new StringTokenizer(normalizedLine); // NOPMD
|
||||
if (!tokenizer.hasMoreTokens()) {
|
||||
final String[] rawColumns = normalizedLine.split("\t", -1);
|
||||
if (rawColumns.length == 0) {
|
||||
ignoredLineCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final String stem = tokenizer.nextToken();
|
||||
final String[] variants = new String[tokenizer.countTokens()]; // NOPMD
|
||||
final String stem = rawColumns[0].strip();
|
||||
final List<String> acceptedVariants = new ArrayList<String>(Math.max(0, rawColumns.length - 1)); // NOPMD
|
||||
|
||||
for (int index = 0; index < variants.length; index++) {
|
||||
variants[index] = tokenizer.nextToken();
|
||||
if (stem.isEmpty()) {
|
||||
ignoredLineCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
entryHandler.onEntry(stem, variants, lineNumber);
|
||||
if (containsWhitespaceCharacter(stem)) {
|
||||
if (LOGGER.isLoggable(Level.WARNING)) {
|
||||
LOGGER.log(Level.WARNING,
|
||||
"Ignoring dictionary line containing whitespace in source {0} at line {1}, stem {2}.",
|
||||
new Object[] { sourceDescription, lineNumber, stem }); // NOPMD
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
int ignored = 0;
|
||||
|
||||
for (int index = 1; index < rawColumns.length; index++) {
|
||||
final String variant = rawColumns[index].strip();
|
||||
if (variant.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (containsWhitespaceCharacter(variant)) {
|
||||
ignored++;
|
||||
continue;
|
||||
}
|
||||
acceptedVariants.add(variant);
|
||||
}
|
||||
|
||||
if (ignored > 0 && LOGGER.isLoggable(Level.WARNING)) {
|
||||
LOGGER.log(Level.WARNING,
|
||||
"Ignoring dictionary items containing whitespace in source {0} at line {1}, stem {2}, ignored {3}:{4}.",
|
||||
new Object[] { sourceDescription, lineNumber, stem, ignored, rawColumns.length }); // NOPMD
|
||||
}
|
||||
|
||||
entryHandler.onEntry(stem, acceptedVariants.toArray(String[]::new), lineNumber);
|
||||
logicalEntryCount++;
|
||||
}
|
||||
|
||||
@@ -188,6 +275,36 @@ public final class StemmerDictionaryParser {
|
||||
return statistics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies case normalization to one line according to the selected mode.
|
||||
*
|
||||
* @param line line to normalize
|
||||
* @param caseProcessingMode case processing mode
|
||||
* @return normalized line
|
||||
*/
|
||||
private static String normalizeLineCase(final String line, final CaseProcessingMode caseProcessingMode) {
|
||||
if (caseProcessingMode == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT) {
|
||||
return line.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether one dictionary item contains any Unicode whitespace
|
||||
* character.
|
||||
*
|
||||
* @param item dictionary item to inspect
|
||||
* @return {@code true} when the item contains at least one whitespace character
|
||||
*/
|
||||
private static boolean containsWhitespaceCharacter(final String item) {
|
||||
for (int index = 0; index < item.length(); index++) {
|
||||
if (Character.isWhitespace(item.charAt(index))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a trailing remark from one physical line.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,758 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.Reader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.SplittableRandom;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Evaluates how stemming quality degrades when the compiled trie is built from
|
||||
* only a deterministic subset of the available dictionary knowledge.
|
||||
*
|
||||
* <p>
|
||||
* The experiment operates on whole dictionary entries. For a chosen knowledge
|
||||
* percentage, each parsed dictionary line is deterministically included or
|
||||
* excluded from the training subset using a seeded {@link SplittableRandom}.
|
||||
* The resulting subset is compiled into a {@link FrequencyTrie}, while the
|
||||
* evaluation is performed against all word forms from the original dictionary.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Two lookup APIs are evaluated:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>{@link FrequencyTrie#get(String)} through top-1 accuracy</li>
|
||||
* <li>{@link FrequencyTrie#getAll(String)} through global precision, recall,
|
||||
* and F1</li>
|
||||
* </ul>
|
||||
*/
|
||||
public final class StemmerKnowledgeExperiment {
|
||||
|
||||
/**
|
||||
* Logger of this class.
|
||||
*/
|
||||
private static final Logger LOGGER = Logger.getLogger(StemmerKnowledgeExperiment.class.getName());
|
||||
|
||||
/**
|
||||
* Minimum supported knowledge percentage.
|
||||
*/
|
||||
public static final int MINIMUM_KNOWLEDGE_PERCENT = 10;
|
||||
|
||||
/**
|
||||
* Maximum supported knowledge percentage.
|
||||
*/
|
||||
public static final int MAXIMUM_KNOWLEDGE_PERCENT = 100;
|
||||
|
||||
/**
|
||||
* Step between adjacent evaluated knowledge percentages.
|
||||
*/
|
||||
public static final int KNOWLEDGE_PERCENT_STEP = 10;
|
||||
|
||||
/**
|
||||
* Canonical no-op patch command.
|
||||
*/
|
||||
private static final String NOOP_PATCH_COMMAND = PatchCommandEncoder.NOOP_PATCH;
|
||||
|
||||
/**
|
||||
* Shared patch encoder reused for subset compilation.
|
||||
*/
|
||||
private final PatchCommandEncoder patchCommandEncoder;
|
||||
|
||||
/**
|
||||
* Creates a new experiment harness.
|
||||
*/
|
||||
public StemmerKnowledgeExperiment() {
|
||||
this.patchCommandEncoder = PatchCommandEncoder.builder().build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates all supported bundled dictionaries using the supplied seed.
|
||||
*
|
||||
* @param seed deterministic sampling seed
|
||||
* @return immutable ordered list of experiment rows
|
||||
* @throws IOException if reading a bundled dictionary fails
|
||||
*/
|
||||
public List<ResultRow> evaluateAllBundledLanguages(final long seed) throws IOException {
|
||||
final List<ResultRow> rows = new ArrayList<>();
|
||||
for (StemmerPatchTrieLoader.Language language : StemmerPatchTrieLoader.Language.values()) {
|
||||
rows.addAll(evaluateBundledLanguage(language, seed));
|
||||
}
|
||||
return List.copyOf(rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates one bundled dictionary across all supported experiment
|
||||
* configurations.
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param seed deterministic sampling seed
|
||||
* @return immutable ordered list of experiment rows
|
||||
* @throws NullPointerException if {@code language} is {@code null}
|
||||
* @throws IOException if reading the bundled dictionary fails
|
||||
*/
|
||||
public List<ResultRow> evaluateBundledLanguage(final StemmerPatchTrieLoader.Language language, final long seed)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
final String resourcePath = language.resourcePath();
|
||||
try (InputStream inputStream = StemmerPatchTrieLoader.openBundledResource(resourcePath)) {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
return evaluate(reader, resourcePath, language.name(), seed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates one filesystem dictionary across all supported experiment
|
||||
* configurations.
|
||||
*
|
||||
* @param dictionaryPath path to a dictionary file
|
||||
* @param seed deterministic sampling seed
|
||||
* @return immutable ordered list of experiment rows
|
||||
* @throws NullPointerException if {@code dictionaryPath} is {@code null}
|
||||
* @throws IOException if reading fails
|
||||
*/
|
||||
public List<ResultRow> evaluatePath(final Path dictionaryPath, final long seed) throws IOException {
|
||||
Objects.requireNonNull(dictionaryPath, "dictionaryPath");
|
||||
try (BufferedReader reader = Files.newBufferedReader(dictionaryPath, StandardCharsets.UTF_8)) {
|
||||
return evaluate(reader, dictionaryPath.toAbsolutePath().toString(), dictionaryPath.getFileName().toString(),
|
||||
seed);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates a dictionary provided through an arbitrary reader.
|
||||
*
|
||||
* @param reader source reader
|
||||
* @param sourceDescription logical source description
|
||||
* @param languageLabel label stored in the result rows
|
||||
* @param seed deterministic sampling seed
|
||||
* @return immutable ordered list of experiment rows
|
||||
* @throws NullPointerException if any argument except {@code seed} is
|
||||
* {@code null}
|
||||
* @throws IOException if parsing fails
|
||||
*/
|
||||
public List<ResultRow> evaluate(final Reader reader, final String sourceDescription, final String languageLabel,
|
||||
final long seed) throws IOException {
|
||||
Objects.requireNonNull(reader, "reader");
|
||||
Objects.requireNonNull(sourceDescription, "sourceDescription");
|
||||
Objects.requireNonNull(languageLabel, "languageLabel");
|
||||
|
||||
final DictionaryData dictionaryData = readDictionary(reader, sourceDescription);
|
||||
final List<ResultRow> rows = new ArrayList<>();
|
||||
|
||||
for (ReductionMode reductionMode : ReductionMode.values()) {
|
||||
final ReductionSettings reductionSettings = ReductionSettings.withDefaults(reductionMode);
|
||||
for (boolean storeOriginal : new boolean[] { false, true }) { // NOPMD
|
||||
for (boolean includeStemInEvaluation : new boolean[] { false, true }) { // NOPMD
|
||||
for (int knowledgePercent = MINIMUM_KNOWLEDGE_PERCENT; knowledgePercent <= MAXIMUM_KNOWLEDGE_PERCENT; knowledgePercent += KNOWLEDGE_PERCENT_STEP) {
|
||||
final ResultRow row = evaluateScenario(dictionaryData, languageLabel, seed, reductionSettings,
|
||||
storeOriginal, includeStemInEvaluation, knowledgePercent);
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (LOGGER.isLoggable(Level.INFO)) {
|
||||
LOGGER.log(Level.INFO, "Knowledge experiment finished for source {0}: entries={1}, rows={2}, seed={3}.",
|
||||
new Object[] { sourceDescription, dictionaryData.entryCount(), rows.size(), seed });
|
||||
}
|
||||
|
||||
return List.copyOf(rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes result rows as UTF-8 CSV with a stable fixed header.
|
||||
*
|
||||
* @param outputPath target file path
|
||||
* @param rows rows to write
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if writing fails
|
||||
*/
|
||||
public static void writeCsv(final Path outputPath, final List<ResultRow> rows) throws IOException {
|
||||
Objects.requireNonNull(outputPath, "outputPath");
|
||||
Objects.requireNonNull(rows, "rows");
|
||||
|
||||
final Path parent = outputPath.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
|
||||
final List<String> lines = new ArrayList<>(rows.size() + 1);
|
||||
lines.add(ResultRow.csvHeader());
|
||||
for (ResultRow row : rows) {
|
||||
lines.add(row.toCsvRow());
|
||||
}
|
||||
Files.write(outputPath, lines, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the full dictionary into an in-memory representation suitable for
|
||||
* repeated deterministic subset compilation.
|
||||
*
|
||||
* @param reader source reader
|
||||
* @param sourceDescription logical source description
|
||||
* @return parsed dictionary data
|
||||
* @throws IOException if parsing fails
|
||||
*/
|
||||
private static DictionaryData readDictionary(final Reader reader, final String sourceDescription)
|
||||
throws IOException {
|
||||
final List<DictionaryEntry> entries = new ArrayList<>();
|
||||
final StemmerDictionaryParser.ParseStatistics parseStatistics = StemmerDictionaryParser.parse(reader,
|
||||
sourceDescription,
|
||||
(stem, variants, lineNumber) -> entries.add(new DictionaryEntry(stem, variants, lineNumber)));
|
||||
return new DictionaryData(sourceDescription, parseStatistics, entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates one concrete experiment scenario.
|
||||
*
|
||||
* @param dictionaryData parsed dictionary data
|
||||
* @param languageLabel logical language label
|
||||
* @param seed deterministic sampling seed
|
||||
* @param reductionSettings reduction settings
|
||||
* @param storeOriginal whether canonical stems are inserted with a
|
||||
* no-op patch
|
||||
* @param includeStemInEvaluation whether the canonical stem itself is evaluated
|
||||
* @param knowledgePercent retained percentage of dictionary entries
|
||||
* @return result row
|
||||
*/
|
||||
private ResultRow evaluateScenario(final DictionaryData dictionaryData, final String languageLabel, final long seed,
|
||||
final ReductionSettings reductionSettings, final boolean storeOriginal,
|
||||
final boolean includeStemInEvaluation, final int knowledgePercent) {
|
||||
final FrequencyTrie<String> trie = compileSubset(dictionaryData, reductionSettings, storeOriginal,
|
||||
knowledgePercent, seed);
|
||||
|
||||
long evaluatedInputCount = 0L;
|
||||
long getCorrectCount = 0L;
|
||||
long getAllTruePositiveCount = 0L;
|
||||
long getAllFalsePositiveCount = 0L;
|
||||
long getAllCoveredInputCount = 0L;
|
||||
long uniqueCandidateCount = 0L;
|
||||
|
||||
for (DictionaryEntry entry : dictionaryData.entries()) {
|
||||
if (includeStemInEvaluation) {
|
||||
final EvaluationCounts stemCounts = evaluateInput(entry.stem(), entry.stem(), trie);
|
||||
evaluatedInputCount++;
|
||||
getCorrectCount += stemCounts.getCorrect();
|
||||
getAllTruePositiveCount += stemCounts.getAllTruePositives();
|
||||
getAllFalsePositiveCount += stemCounts.getAllFalsePositives();
|
||||
getAllCoveredInputCount += stemCounts.getAllCoveredInputs();
|
||||
uniqueCandidateCount += stemCounts.getUniqueCandidateCount();
|
||||
}
|
||||
for (String variant : entry.variants()) {
|
||||
final EvaluationCounts variantCounts = evaluateInput(variant, entry.stem(), trie);
|
||||
evaluatedInputCount++;
|
||||
getCorrectCount += variantCounts.getCorrect();
|
||||
getAllTruePositiveCount += variantCounts.getAllTruePositives();
|
||||
getAllFalsePositiveCount += variantCounts.getAllFalsePositives();
|
||||
getAllCoveredInputCount += variantCounts.getAllCoveredInputs();
|
||||
uniqueCandidateCount += variantCounts.getUniqueCandidateCount();
|
||||
}
|
||||
}
|
||||
|
||||
final long trainingEntryCount = countSelectedEntries(dictionaryData.entryCount(), seed, knowledgePercent);
|
||||
final double getAccuracy = ratio(getCorrectCount, evaluatedInputCount);
|
||||
final double getAllPrecision = ratio(getAllTruePositiveCount,
|
||||
getAllTruePositiveCount + getAllFalsePositiveCount);
|
||||
final double getAllRecall = ratio(getAllCoveredInputCount, evaluatedInputCount);
|
||||
final double getAllF1 = f1(getAllPrecision, getAllRecall);
|
||||
final double averageUniqueCandidateCount = ratio(uniqueCandidateCount, evaluatedInputCount);
|
||||
|
||||
return new ResultRow(languageLabel, reductionSettings.reductionMode().name(), storeOriginal,
|
||||
includeStemInEvaluation, knowledgePercent, seed, dictionaryData.entryCount(), trainingEntryCount,
|
||||
evaluatedInputCount, getCorrectCount, getAccuracy, getAllTruePositiveCount, getAllFalsePositiveCount,
|
||||
getAllCoveredInputCount, getAllPrecision, getAllRecall, getAllF1, averageUniqueCandidateCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compiles a trie from the deterministically selected subset of dictionary
|
||||
* entries.
|
||||
*
|
||||
* @param dictionaryData parsed dictionary data
|
||||
* @param reductionSettings reduction settings
|
||||
* @param storeOriginal whether stems themselves should be stored
|
||||
* @param knowledgePercent retained percentage of dictionary entries
|
||||
* @param seed deterministic sampling seed
|
||||
* @return compiled trie for the selected subset
|
||||
*/
|
||||
private FrequencyTrie<String> compileSubset(final DictionaryData dictionaryData,
|
||||
final ReductionSettings reductionSettings, final boolean storeOriginal, final int knowledgePercent,
|
||||
final long seed) {
|
||||
validateKnowledgePercent(knowledgePercent);
|
||||
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, reductionSettings);
|
||||
final SplittableRandom random = new SplittableRandom(seed);
|
||||
|
||||
for (DictionaryEntry entry : dictionaryData.entries()) {
|
||||
if (!isSelected(random, knowledgePercent)) {
|
||||
continue;
|
||||
}
|
||||
if (storeOriginal) {
|
||||
builder.put(entry.stem(), NOOP_PATCH_COMMAND);
|
||||
}
|
||||
for (String variant : entry.variants()) {
|
||||
final String patch = this.patchCommandEncoder.encode(variant, entry.stem());
|
||||
builder.put(variant, patch);
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates one input word form against both lookup APIs.
|
||||
*
|
||||
* @param input input form to transform
|
||||
* @param expectedStem expected stem
|
||||
* @param trie compiled trie under test
|
||||
* @return immutable counts for this single input
|
||||
*/
|
||||
private static EvaluationCounts evaluateInput(final String input, final String expectedStem,
|
||||
final FrequencyTrie<String> trie) {
|
||||
long getCorrect = 0L;
|
||||
final String preferredPatch = trie.get(input);
|
||||
if (preferredPatch != null) {
|
||||
final String preferredStem = PatchCommandEncoder.apply(input, preferredPatch);
|
||||
if (expectedStem.equals(preferredStem)) {
|
||||
getCorrect = 1L;
|
||||
}
|
||||
} else {
|
||||
if (expectedStem.equals(input)) {
|
||||
getCorrect = 1L;
|
||||
}
|
||||
}
|
||||
|
||||
final String[] patches = trie.getAll(input);
|
||||
|
||||
long truePositives = 0L;
|
||||
long falsePositives = 0L;
|
||||
long coveredInputs = 0L;
|
||||
for (String patch : patches) {
|
||||
final String candidateStem = PatchCommandEncoder.apply(input, patch);
|
||||
if (expectedStem.equals(candidateStem)) {
|
||||
truePositives++;
|
||||
coveredInputs = 1L;
|
||||
} else {
|
||||
falsePositives++;
|
||||
}
|
||||
}
|
||||
return new EvaluationCounts(getCorrect, truePositives, falsePositives, coveredInputs, patches.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts how many entries would be selected for one scenario without
|
||||
* recompiling the trie.
|
||||
*
|
||||
* @param entryCount total entry count
|
||||
* @param seed deterministic sampling seed
|
||||
* @param knowledgePercent retained percentage of dictionary entries
|
||||
* @return selected entry count
|
||||
*/
|
||||
private static long countSelectedEntries(final int entryCount, final long seed, final int knowledgePercent) {
|
||||
validateKnowledgePercent(knowledgePercent);
|
||||
final SplittableRandom random = new SplittableRandom(seed);
|
||||
long count = 0L;
|
||||
for (int index = 0; index < entryCount; index++) {
|
||||
if (isSelected(random, knowledgePercent)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether one entry is selected for the supplied knowledge level.
|
||||
*
|
||||
* @param random deterministic random source
|
||||
* @param knowledgePercent retained percentage of entries
|
||||
* @return {@code true} when the entry should be kept
|
||||
*/
|
||||
private static boolean isSelected(final SplittableRandom random, final int knowledgePercent) {
|
||||
return random.nextInt(100) < knowledgePercent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates one knowledge percentage value.
|
||||
*
|
||||
* @param knowledgePercent value to validate
|
||||
*/
|
||||
private static void validateKnowledgePercent(final int knowledgePercent) {
|
||||
if (knowledgePercent < MINIMUM_KNOWLEDGE_PERCENT || knowledgePercent > MAXIMUM_KNOWLEDGE_PERCENT
|
||||
|| knowledgePercent % KNOWLEDGE_PERCENT_STEP != 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"knowledgePercent must be one of 10, 20, ..., 100 but was " + knowledgePercent + '.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a safe ratio.
|
||||
*
|
||||
* @param numerator numerator
|
||||
* @param denominator denominator
|
||||
* @return ratio, or {@code 0.0} when the denominator is zero
|
||||
*/
|
||||
private static double ratio(final long numerator, final long denominator) {
|
||||
if (denominator == 0L) { // NOPMD
|
||||
return 0.0d;
|
||||
}
|
||||
return (double) numerator / (double) denominator; // NOPMD
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the harmonic mean of precision and recall.
|
||||
*
|
||||
* @param precision global precision
|
||||
* @param recall global recall
|
||||
* @return F1 score, or {@code 0.0} when both inputs are zero
|
||||
*/
|
||||
private static double f1(final double precision, final double recall) {
|
||||
if (precision == 0.0d && recall == 0.0d) {
|
||||
return 0.0d;
|
||||
}
|
||||
return 2.0d * precision * recall / (precision + recall);
|
||||
}
|
||||
|
||||
/**
|
||||
* One parsed dictionary line.
|
||||
*
|
||||
* @param stem canonical stem
|
||||
* @param variants known variants of the stem
|
||||
* @param lineNumber physical line number in the source dictionary
|
||||
*/
|
||||
private record DictionaryEntry(String stem, String[] variants, int lineNumber) {
|
||||
|
||||
/**
|
||||
* Creates a parsed dictionary entry.
|
||||
*
|
||||
* @param stem canonical stem
|
||||
* @param variants known variants of the stem
|
||||
* @param lineNumber physical line number in the source dictionary
|
||||
*/
|
||||
private DictionaryEntry {
|
||||
Objects.requireNonNull(stem, "stem");
|
||||
Objects.requireNonNull(variants, "variants");
|
||||
if (lineNumber < 1) { // NOPMD
|
||||
throw new IllegalArgumentException("lineNumber must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed dictionary state reused across all scenarios.
|
||||
*
|
||||
* @param sourceDescription logical source description
|
||||
* @param parseStatistics parser statistics
|
||||
* @param entries immutable ordered entries
|
||||
*/
|
||||
private record DictionaryData(String sourceDescription, StemmerDictionaryParser.ParseStatistics parseStatistics,
|
||||
List<DictionaryEntry> entries) {
|
||||
|
||||
/**
|
||||
* Creates parsed dictionary data.
|
||||
*
|
||||
* @param sourceDescription logical source description
|
||||
* @param parseStatistics parser statistics
|
||||
* @param entries immutable ordered entries
|
||||
*/
|
||||
private DictionaryData {
|
||||
Objects.requireNonNull(sourceDescription, "sourceDescription");
|
||||
Objects.requireNonNull(parseStatistics, "parseStatistics");
|
||||
Objects.requireNonNull(entries, "entries");
|
||||
entries = List.copyOf(entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of logical dictionary entries.
|
||||
*
|
||||
* @return entry count
|
||||
*/
|
||||
private int entryCount() {
|
||||
return this.entries.size();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-input evaluation counts.
|
||||
*/
|
||||
private static final class EvaluationCounts {
|
||||
|
||||
/**
|
||||
* Preferred lookup correctness.
|
||||
*/
|
||||
private final long getCorrect;
|
||||
|
||||
/**
|
||||
* Number of correct candidates returned by {@code getAll()}.
|
||||
*/
|
||||
private final long getAllTruePositives;
|
||||
|
||||
/**
|
||||
* Number of incorrect candidates returned by {@code getAll()}.
|
||||
*/
|
||||
private final long getAllFalsePositives;
|
||||
|
||||
/**
|
||||
* Whether the correct stem was covered by {@code getAll()}.
|
||||
*/
|
||||
private final long getAllCoveredInputs;
|
||||
|
||||
/**
|
||||
* Number of candidate commands returned by {@code getAll()}.
|
||||
*/
|
||||
private final long uniqueCandidateCount;
|
||||
|
||||
/**
|
||||
* Creates a new immutable counter object.
|
||||
*
|
||||
* @param getCorrect preferred lookup correctness
|
||||
* @param getAllTruePositives correct candidates
|
||||
* @param getAllFalsePositives incorrect candidates
|
||||
* @param getAllCoveredInputs coverage marker
|
||||
* @param uniqueCandidateCount candidate command count
|
||||
*/
|
||||
private EvaluationCounts(final long getCorrect, final long getAllTruePositives, final long getAllFalsePositives,
|
||||
final long getAllCoveredInputs, final long uniqueCandidateCount) {
|
||||
this.getCorrect = getCorrect;
|
||||
this.getAllTruePositives = getAllTruePositives;
|
||||
this.getAllFalsePositives = getAllFalsePositives;
|
||||
this.getAllCoveredInputs = getAllCoveredInputs;
|
||||
this.uniqueCandidateCount = uniqueCandidateCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns preferred lookup correctness.
|
||||
*
|
||||
* @return preferred lookup correctness
|
||||
*/
|
||||
private long getCorrect() {
|
||||
return this.getCorrect;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of correct candidates.
|
||||
*
|
||||
* @return correct candidates
|
||||
*/
|
||||
private long getAllTruePositives() {
|
||||
return this.getAllTruePositives;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of incorrect candidates.
|
||||
*
|
||||
* @return incorrect candidates
|
||||
*/
|
||||
private long getAllFalsePositives() {
|
||||
return this.getAllFalsePositives;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the per-input coverage marker.
|
||||
*
|
||||
* @return coverage marker
|
||||
*/
|
||||
private long getAllCoveredInputs() {
|
||||
return this.getAllCoveredInputs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of candidate commands.
|
||||
*
|
||||
* @return candidate command count
|
||||
*/
|
||||
private long getUniqueCandidateCount() {
|
||||
return this.uniqueCandidateCount;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One immutable result row of the knowledge experiment.
|
||||
*
|
||||
* @param language language label
|
||||
* @param reductionMode reduction mode name
|
||||
* @param storeOriginal whether no-op patches were stored for
|
||||
* canonical stems
|
||||
* @param includeStemInEvaluation whether canonical stems were part of the
|
||||
* evaluated inputs
|
||||
* @param knowledgePercent retained knowledge percentage
|
||||
* @param seed deterministic sampling seed
|
||||
* @param dictionaryEntryCount total parsed dictionary entry count
|
||||
* @param trainingEntryCount selected dictionary entry count used for
|
||||
* build
|
||||
* @param evaluatedInputCount total evaluated input count
|
||||
* @param getCorrectCount number of correct preferred
|
||||
* transformations
|
||||
* @param getAccuracy preferred lookup accuracy
|
||||
* @param getAllTruePositiveCount number of unique correct candidates from
|
||||
* {@code getAll()}
|
||||
* @param getAllFalsePositiveCount number of unique incorrect candidates from
|
||||
* {@code getAll()}
|
||||
* @param getAllCoveredInputCount number of inputs for which the correct
|
||||
* stem appeared in {@code getAll()}
|
||||
* @param getAllPrecision global candidate precision for
|
||||
* {@code getAll()}
|
||||
* @param getAllRecall global input recall for {@code getAll()}
|
||||
* @param getAllF1 F1 score derived from {@code getAll()}
|
||||
* precision and recall
|
||||
* @param averageUniqueCandidateCount average number of unique candidate stems
|
||||
* per input
|
||||
*/
|
||||
public record ResultRow(String language, String reductionMode, boolean storeOriginal,
|
||||
boolean includeStemInEvaluation, int knowledgePercent, long seed, int dictionaryEntryCount,
|
||||
long trainingEntryCount, long evaluatedInputCount, long getCorrectCount, double getAccuracy,
|
||||
long getAllTruePositiveCount, long getAllFalsePositiveCount, long getAllCoveredInputCount,
|
||||
double getAllPrecision, double getAllRecall, double getAllF1, double averageUniqueCandidateCount) {
|
||||
|
||||
/**
|
||||
* Creates one immutable result row.
|
||||
*
|
||||
* @param language language label
|
||||
* @param reductionMode reduction mode name
|
||||
* @param storeOriginal whether no-op patches were stored for
|
||||
* canonical stems
|
||||
* @param includeStemInEvaluation whether canonical stems were evaluated
|
||||
* @param knowledgePercent retained knowledge percentage
|
||||
* @param seed deterministic sampling seed
|
||||
* @param dictionaryEntryCount total dictionary entry count
|
||||
* @param trainingEntryCount selected training entry count
|
||||
* @param evaluatedInputCount total evaluated input count
|
||||
* @param getCorrectCount number of correct preferred
|
||||
* transformations
|
||||
* @param getAccuracy preferred lookup accuracy
|
||||
* @param getAllTruePositiveCount number of unique correct candidates
|
||||
* @param getAllFalsePositiveCount number of unique incorrect candidates
|
||||
* @param getAllCoveredInputCount coverage count for {@code getAll()}
|
||||
* @param getAllPrecision global candidate precision for
|
||||
* {@code getAll()}
|
||||
* @param getAllRecall global input recall for {@code getAll()}
|
||||
* @param getAllF1 harmonic mean of precision and recall
|
||||
* @param averageUniqueCandidateCount average unique candidate count per input
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
public ResultRow {
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(reductionMode, "reductionMode");
|
||||
validateKnowledgePercent(knowledgePercent);
|
||||
if (dictionaryEntryCount < 0) {
|
||||
throw new IllegalArgumentException("dictionaryEntryCount must not be negative.");
|
||||
}
|
||||
if (trainingEntryCount < 0L) {
|
||||
throw new IllegalArgumentException("trainingEntryCount must not be negative.");
|
||||
}
|
||||
if (evaluatedInputCount < 0L) {
|
||||
throw new IllegalArgumentException("evaluatedInputCount must not be negative.");
|
||||
}
|
||||
if (getCorrectCount < 0L) {
|
||||
throw new IllegalArgumentException("getCorrectCount must not be negative.");
|
||||
}
|
||||
if (getAllTruePositiveCount < 0L) {
|
||||
throw new IllegalArgumentException("getAllTruePositiveCount must not be negative.");
|
||||
}
|
||||
if (getAllFalsePositiveCount < 0L) {
|
||||
throw new IllegalArgumentException("getAllFalsePositiveCount must not be negative.");
|
||||
}
|
||||
if (getAllCoveredInputCount < 0L) {
|
||||
throw new IllegalArgumentException("getAllCoveredInputCount must not be negative.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the stable CSV header of this result format.
|
||||
*
|
||||
* @return CSV header line
|
||||
*/
|
||||
public static String csvHeader() {
|
||||
return String.join(",",
|
||||
List.of("language", "reductionMode", "storeOriginal", "includeStemInEvaluation", "knowledgePercent",
|
||||
"seed", "dictionaryEntryCount", "trainingEntryCount", "evaluatedInputCount",
|
||||
"getCorrectCount", "getAccuracy", "getAllTruePositiveCount", "getAllFalsePositiveCount",
|
||||
"getAllCoveredInputCount", "getAllPrecision", "getAllRecall", "getAllF1",
|
||||
"averageUniqueCandidateCount"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes this row as one CSV record.
|
||||
*
|
||||
* @return CSV record
|
||||
*/
|
||||
public String toCsvRow() {
|
||||
return String.join(",",
|
||||
List.of(escapeCsv(this.language), escapeCsv(this.reductionMode), String.valueOf(this.storeOriginal),
|
||||
String.valueOf(this.includeStemInEvaluation), String.valueOf(this.knowledgePercent),
|
||||
String.valueOf(this.seed), String.valueOf(this.dictionaryEntryCount),
|
||||
String.valueOf(this.trainingEntryCount), String.valueOf(this.evaluatedInputCount),
|
||||
String.valueOf(this.getCorrectCount), formatDouble(this.getAccuracy),
|
||||
String.valueOf(this.getAllTruePositiveCount), String.valueOf(this.getAllFalsePositiveCount),
|
||||
String.valueOf(this.getAllCoveredInputCount), formatDouble(this.getAllPrecision),
|
||||
formatDouble(this.getAllRecall), formatDouble(this.getAllF1),
|
||||
formatDouble(this.averageUniqueCandidateCount)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes a string for CSV output.
|
||||
*
|
||||
* @param value value to escape
|
||||
* @return escaped CSV cell
|
||||
*/
|
||||
private static String escapeCsv(final String value) {
|
||||
if (value.indexOf(',') < 0 && value.indexOf('"') < 0 && value.indexOf('\n') < 0
|
||||
&& value.indexOf('\r') < 0) {
|
||||
return value;
|
||||
}
|
||||
return '"' + value.replace("\"", "\"\"") + '"';
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats one floating-point value using a locale-independent decimal
|
||||
* representation.
|
||||
*
|
||||
* @param value value to format
|
||||
* @return formatted value
|
||||
*/
|
||||
private static String formatDouble(final double value) {
|
||||
return String.format(Locale.ROOT, "%.10f", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Command-line entry point for the stemmer knowledge experiment.
|
||||
*/
|
||||
public final class StemmerKnowledgeExperimentCli {
|
||||
|
||||
/**
|
||||
* Logger of this class.
|
||||
*/
|
||||
private static final Logger LOGGER = Logger.getLogger(StemmerKnowledgeExperimentCli.class.getName());
|
||||
|
||||
/**
|
||||
* Exit status indicating success.
|
||||
*/
|
||||
private static final int EXIT_SUCCESS = 0;
|
||||
|
||||
/**
|
||||
* Exit status indicating processing failure.
|
||||
*/
|
||||
private static final int EXIT_PROCESSING_ERROR = 1;
|
||||
|
||||
/**
|
||||
* Exit status indicating invalid command-line usage.
|
||||
*/
|
||||
private static final int EXIT_USAGE_ERROR = 2;
|
||||
|
||||
/**
|
||||
* Default deterministic seed.
|
||||
*/
|
||||
private static final long DEFAULT_SEED = 20_260_421L;
|
||||
|
||||
/**
|
||||
* Default output report location.
|
||||
*/
|
||||
private static final Path DEFAULT_OUTPUT_PATH = Path.of("build", "reports", "stemmer-knowledge-experiment.csv");
|
||||
|
||||
/**
|
||||
* Usage banner.
|
||||
*/
|
||||
private static final String USAGE = String.join(System.lineSeparator(),
|
||||
"Usage: StemmerKnowledgeExperimentCli [--bundled-all | --bundled-language <LANG> | --input <PATH>]",
|
||||
" [--seed <LONG>] [--output <CSV_PATH>]", "", "Examples:", " --bundled-all",
|
||||
" --bundled-language US_UK_PROFI --seed 20260421",
|
||||
" --input src/main/resources/us_uk/stemmer --output build/reports/knowledge.csv");
|
||||
|
||||
/**
|
||||
* Utility class.
|
||||
*/
|
||||
private StemmerKnowledgeExperimentCli() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the CLI as a standalone process.
|
||||
*
|
||||
* @param arguments command-line arguments
|
||||
*/
|
||||
public static void main(final String[] arguments) {
|
||||
final int exitCode = execute(arguments);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the CLI and translates all outcomes to process exit codes.
|
||||
*
|
||||
* @param arguments command-line arguments
|
||||
* @return process exit code
|
||||
*/
|
||||
/* default */ static int execute(final String... arguments) {
|
||||
Objects.requireNonNull(arguments, "arguments");
|
||||
try {
|
||||
final CliOptions options = CliOptions.parse(arguments);
|
||||
if (options.command() == Command.HELP) {
|
||||
printUsage(System.out);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
return runExperiment(options);
|
||||
} catch (final CliUsageException exception) {
|
||||
if (LOGGER.isLoggable(Level.SEVERE)) {
|
||||
LOGGER.log(Level.SEVERE, "Invalid command-line usage for arguments {0}: {1}",
|
||||
new Object[] { Arrays.toString(arguments), exception.getMessage() });
|
||||
}
|
||||
printUsage(System.err);
|
||||
return EXIT_USAGE_ERROR;
|
||||
} catch (final IOException exception) {
|
||||
if (LOGGER.isLoggable(Level.SEVERE)) {
|
||||
LOGGER.log(Level.SEVERE, "Experiment processing failed for arguments {0}", Arrays.toString(arguments));
|
||||
LOGGER.log(Level.SEVERE, "Processing failure details.", exception);
|
||||
}
|
||||
return EXIT_PROCESSING_ERROR;
|
||||
} catch (final RuntimeException exception) { // NOPMD
|
||||
if (LOGGER.isLoggable(Level.SEVERE)) {
|
||||
LOGGER.log(Level.SEVERE, "Unexpected runtime failure for arguments {0}", Arrays.toString(arguments));
|
||||
LOGGER.log(Level.SEVERE, "Unexpected processing failure details.", exception);
|
||||
}
|
||||
return EXIT_PROCESSING_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the experiment for already validated options.
|
||||
*
|
||||
* @param options validated CLI options
|
||||
* @return process exit code
|
||||
* @throws IOException if experiment execution fails
|
||||
*/
|
||||
private static int runExperiment(final CliOptions options) throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = switch (options.sourceMode()) {
|
||||
case INPUT_PATH -> experiment.evaluatePath(options.inputPath(), options.seed());
|
||||
case SINGLE_BUNDLED_LANGUAGE -> experiment.evaluateBundledLanguage(options.language(), options.seed());
|
||||
case ALL_BUNDLED_LANGUAGES -> experiment.evaluateAllBundledLanguages(options.seed());
|
||||
};
|
||||
|
||||
StemmerKnowledgeExperiment.writeCsv(options.outputPath(), rows);
|
||||
if (LOGGER.isLoggable(Level.INFO)) {
|
||||
LOGGER.log(Level.INFO, "Knowledge experiment report written to {0} with {1} rows.",
|
||||
new Object[] { options.outputPath().toAbsolutePath(), rows.size() });
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints the CLI usage text.
|
||||
*
|
||||
* @param stream target output stream
|
||||
*/
|
||||
private static void printUsage(final PrintStream stream) {
|
||||
stream.println(USAGE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Supported top-level CLI commands.
|
||||
*/
|
||||
private enum Command {
|
||||
|
||||
/**
|
||||
* Executes the experiment.
|
||||
*/
|
||||
EXECUTE,
|
||||
|
||||
/**
|
||||
* Prints usage text.
|
||||
*/
|
||||
HELP
|
||||
}
|
||||
|
||||
/**
|
||||
* Supported experiment source selection modes.
|
||||
*/
|
||||
private enum ExperimentSourceMode {
|
||||
|
||||
/**
|
||||
* Runs the experiment for all bundled languages.
|
||||
*/
|
||||
ALL_BUNDLED_LANGUAGES,
|
||||
|
||||
/**
|
||||
* Runs the experiment for one bundled language.
|
||||
*/
|
||||
SINGLE_BUNDLED_LANGUAGE,
|
||||
|
||||
/**
|
||||
* Runs the experiment for one external dictionary path.
|
||||
*/
|
||||
INPUT_PATH
|
||||
}
|
||||
|
||||
/**
|
||||
* Exception indicating invalid command-line usage.
|
||||
*/
|
||||
private static final class CliUsageException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = -3904751711104596247L;
|
||||
|
||||
/**
|
||||
* Creates a new usage exception.
|
||||
*
|
||||
* @param message failure description
|
||||
*/
|
||||
private CliUsageException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new usage exception.
|
||||
*
|
||||
* @param message failure description
|
||||
* @param cause original cause
|
||||
*/
|
||||
private CliUsageException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed CLI options.
|
||||
*
|
||||
* @param command selected top-level command
|
||||
* @param sourceMode selected experiment source mode
|
||||
* @param inputPath optional filesystem dictionary path
|
||||
* @param language optional bundled language
|
||||
* @param seed deterministic sampling seed
|
||||
* @param outputPath CSV report path
|
||||
*/
|
||||
private record CliOptions(Command command, ExperimentSourceMode sourceMode, Path inputPath,
|
||||
StemmerPatchTrieLoader.Language language, long seed, Path outputPath) {
|
||||
|
||||
/**
|
||||
* Parses the command line.
|
||||
*
|
||||
* @param arguments command-line arguments
|
||||
* @return parsed options
|
||||
* @throws CliUsageException if the command line is invalid
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidReassigningLoopVariables")
|
||||
private static CliOptions parse(final String... arguments) throws CliUsageException {
|
||||
Objects.requireNonNull(arguments, "arguments");
|
||||
|
||||
Command command = Command.EXECUTE;
|
||||
ExperimentSourceMode sourceMode = ExperimentSourceMode.ALL_BUNDLED_LANGUAGES;
|
||||
Path inputPath = null;
|
||||
StemmerPatchTrieLoader.Language language = null;
|
||||
long seed = DEFAULT_SEED;
|
||||
Path outputPath = DEFAULT_OUTPUT_PATH;
|
||||
|
||||
final List<String> tokens = new ArrayList<>(List.of(arguments));
|
||||
for (int index = 0; index < tokens.size(); index++) {
|
||||
final String token = tokens.get(index);
|
||||
switch (token) {
|
||||
case "--input" -> {
|
||||
sourceMode = ExperimentSourceMode.INPUT_PATH;
|
||||
inputPath = Path.of(requireValue(tokens, ++index, token));
|
||||
language = null;
|
||||
}
|
||||
case "--bundled-language" -> {
|
||||
sourceMode = ExperimentSourceMode.SINGLE_BUNDLED_LANGUAGE;
|
||||
language = parseLanguage(requireValue(tokens, ++index, token));
|
||||
inputPath = null;
|
||||
}
|
||||
case "--bundled-all" -> {
|
||||
sourceMode = ExperimentSourceMode.ALL_BUNDLED_LANGUAGES;
|
||||
inputPath = null;
|
||||
language = null;
|
||||
}
|
||||
case "--seed" -> seed = parseSeed(requireValue(tokens, ++index, token));
|
||||
case "--output" -> outputPath = Path.of(requireValue(tokens, ++index, token));
|
||||
case "--help", "-h" -> command = Command.HELP;
|
||||
default -> throw new CliUsageException("Unknown argument: " + token);
|
||||
}
|
||||
}
|
||||
|
||||
return new CliOptions(command, sourceMode, inputPath, language, seed, outputPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the required value after one option token.
|
||||
*
|
||||
* @param tokens all tokens
|
||||
* @param index expected value index
|
||||
* @param option current option token
|
||||
* @return option value
|
||||
* @throws CliUsageException if the option value is missing
|
||||
*/
|
||||
private static String requireValue(final List<String> tokens, final int index, final String option)
|
||||
throws CliUsageException {
|
||||
if (index >= tokens.size()) {
|
||||
throw new CliUsageException("Missing value for option " + option + '.');
|
||||
}
|
||||
return tokens.get(index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the deterministic seed.
|
||||
*
|
||||
* @param value textual seed value
|
||||
* @return parsed seed
|
||||
* @throws CliUsageException if the seed value is invalid
|
||||
*/
|
||||
private static long parseSeed(final String value) throws CliUsageException {
|
||||
try {
|
||||
return Long.parseLong(value);
|
||||
} catch (final NumberFormatException exception) {
|
||||
throw new CliUsageException("Invalid value for --seed: " + value, exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the bundled language selector.
|
||||
*
|
||||
* @param value textual language name
|
||||
* @return parsed language
|
||||
* @throws CliUsageException if the language value is invalid
|
||||
*/
|
||||
private static StemmerPatchTrieLoader.Language parseLanguage(final String value) throws CliUsageException {
|
||||
try {
|
||||
return StemmerPatchTrieLoader.Language.valueOf(value);
|
||||
} catch (final IllegalArgumentException exception) {
|
||||
throw new CliUsageException("Invalid value for --bundled-language: " + value, exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,6 +132,48 @@ public final class StemmerPatchTrieBinaryIO {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||
* at a filesystem path.
|
||||
*
|
||||
* @param path source file
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
return read(path).metadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||
* at a filesystem path string.
|
||||
*
|
||||
* @param fileName source file name or path string
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final String fileName) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
return readMetadata(Path.of(fileName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie from an
|
||||
* input stream.
|
||||
*
|
||||
* @param inputStream source stream
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final InputStream inputStream) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
return read(inputStream).metadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a GZip-compressed binary patch-command trie to a filesystem path.
|
||||
*
|
||||
|
||||
@@ -30,24 +30,27 @@
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PushbackInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
/**
|
||||
* Loader of patch-command tries from bundled stemmer dictionaries.
|
||||
*
|
||||
* <p>
|
||||
* Each dictionary is line-oriented. The first token on a line is interpreted as
|
||||
* the stem, and all following tokens are treated as known variants of that
|
||||
* stem.
|
||||
* Each dictionary is line-oriented and uses a tab-separated values layout. The
|
||||
* first column on a line is interpreted as the stem, and all following
|
||||
* tab-separated columns are treated as known variants of that stem.
|
||||
*
|
||||
* <p>
|
||||
* For each line, the loader inserts:
|
||||
@@ -55,15 +58,20 @@ import java.util.logging.Logger;
|
||||
* <li>the stem itself mapped to the canonical no-op patch command
|
||||
* {@link PatchCommandEncoder#NOOP_PATCH}, when requested by the caller</li>
|
||||
* <li>every distinct variant mapped to the patch command transforming that
|
||||
* variant to the stem</li>
|
||||
* variant to the stem using the traversal direction implied by the selected
|
||||
* language or loader overload</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* Parsing is delegated to {@link StemmerDictionaryParser}, which also supports
|
||||
* line remarks introduced by {@code #} or {@code //}.
|
||||
* line remarks introduced by {@code #} or {@code //} and ignores dictionary
|
||||
* items containing Unicode whitespace characters while reporting them through
|
||||
* aggregated warning log records.
|
||||
*/
|
||||
public final class StemmerPatchTrieLoader {
|
||||
|
||||
/* default */ static final String FILENAME_REQUIRED = "fileName required";
|
||||
|
||||
/**
|
||||
* Logger of this class.
|
||||
*/
|
||||
@@ -83,90 +91,151 @@ public final class StemmerPatchTrieLoader {
|
||||
|
||||
/**
|
||||
* Supported bundled stemmer dictionaries.
|
||||
*
|
||||
* <p>
|
||||
* Each language constant defines:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>the resource directory name used under the bundled resources tree</li>
|
||||
* <li>whether the language is written right-to-left</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* The right-to-left flag is intended for consumers that need to decide whether
|
||||
* affix-oriented processing should conceptually traverse words from the visual
|
||||
* end or from the logical beginning of the stored form.
|
||||
* </p>
|
||||
*/
|
||||
public enum Language {
|
||||
|
||||
/**
|
||||
* Czech.
|
||||
*/
|
||||
CS_CZ("cs_cz", false),
|
||||
|
||||
/**
|
||||
* Danish.
|
||||
*/
|
||||
DA_DK("da_dk"),
|
||||
DA_DK("da_dk", false),
|
||||
|
||||
/**
|
||||
* German.
|
||||
*/
|
||||
DE_DE("de_de"),
|
||||
DE_DE("de_de", false),
|
||||
|
||||
/**
|
||||
* Spanish.
|
||||
*/
|
||||
ES_ES("es_es"),
|
||||
ES_ES("es_es", false),
|
||||
|
||||
/**
|
||||
* Persian.
|
||||
*/
|
||||
FA_IR("fa_ir", true),
|
||||
|
||||
/**
|
||||
* Finnish.
|
||||
*/
|
||||
FI_FI("fi_fi", false),
|
||||
|
||||
/**
|
||||
* French.
|
||||
*/
|
||||
FR_FR("fr_fr"),
|
||||
FR_FR("fr_fr", false),
|
||||
|
||||
/**
|
||||
* Hebrew.
|
||||
*/
|
||||
HE_IL("he_il", true),
|
||||
|
||||
/**
|
||||
* Hungarian.
|
||||
*/
|
||||
HU_HU("hu_hu", false),
|
||||
|
||||
/**
|
||||
* Italian.
|
||||
*/
|
||||
IT_IT("it_it"),
|
||||
IT_IT("it_it", false),
|
||||
|
||||
/**
|
||||
* Norwegian Bokmål.
|
||||
*/
|
||||
NB_NO("nb_no", false),
|
||||
|
||||
/**
|
||||
* Dutch.
|
||||
*/
|
||||
NL_NL("nl_nl"),
|
||||
NL_NL("nl_nl", false),
|
||||
|
||||
/**
|
||||
* Norwegian.
|
||||
* Norwegian Nynorsk.
|
||||
*/
|
||||
NO_NO("no_no"),
|
||||
NN_NO("nn_no", false),
|
||||
|
||||
/**
|
||||
* Polish.
|
||||
*/
|
||||
PL_PL("pl_pl", false),
|
||||
|
||||
/**
|
||||
* Portuguese.
|
||||
*/
|
||||
PT_PT("pt_pt"),
|
||||
PT_PT("pt_pt", false),
|
||||
|
||||
/**
|
||||
* Russian.
|
||||
*/
|
||||
RU_RU("ru_ru"),
|
||||
RU_RU("ru_ru", false),
|
||||
|
||||
/**
|
||||
* Swedish.
|
||||
*/
|
||||
SV_SE("sv_se"),
|
||||
SV_SE("sv_se", false),
|
||||
|
||||
/**
|
||||
* Ukrainian.
|
||||
*/
|
||||
UK_UA("uk_ua", false),
|
||||
|
||||
/**
|
||||
* English.
|
||||
*/
|
||||
US_UK("us_uk"),
|
||||
US_UK("us_uk", false),
|
||||
|
||||
/**
|
||||
* English professional dictionary.
|
||||
* Yiddish.
|
||||
*/
|
||||
US_UK_PROFI("us_uk.profi");
|
||||
YI("yi", true);
|
||||
|
||||
/**
|
||||
* Resource directory name.
|
||||
*/
|
||||
private final String resourceDirectory;
|
||||
|
||||
/**
|
||||
* Whether the language is written right-to-left.
|
||||
*/
|
||||
private final boolean rightToLeft;
|
||||
|
||||
/**
|
||||
* Creates a language constant.
|
||||
*
|
||||
* @param resourceDirectory resource directory name
|
||||
* @param rightToLeft whether the language is written right-to-left
|
||||
*/
|
||||
Language(final String resourceDirectory) {
|
||||
Language(final String resourceDirectory, final boolean rightToLeft) {
|
||||
this.resourceDirectory = resourceDirectory;
|
||||
this.rightToLeft = rightToLeft;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the classpath resource path of the stemmer dictionary.
|
||||
* Returns the classpath resource path of the bundled stemmer dictionary.
|
||||
*
|
||||
* @return classpath resource path
|
||||
*/
|
||||
public String resourcePath() {
|
||||
return this.resourceDirectory + "/stemmer";
|
||||
return this.resourceDirectory + "/stemmer.gz";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -177,11 +246,45 @@ public final class StemmerPatchTrieLoader {
|
||||
public String resourceDirectory() {
|
||||
return this.resourceDirectory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the language is written right-to-left.
|
||||
*
|
||||
* <p>
|
||||
* This flag can be used by trie-building and lookup logic to decide whether
|
||||
* suffix-oriented traversal should operate on the stored word form as-is rather
|
||||
* than by reversing the logical character sequence.
|
||||
* </p>
|
||||
*
|
||||
* @return {@code true} when the language is written right-to-left, otherwise
|
||||
* {@code false}
|
||||
*/
|
||||
public boolean isRightToLeft() {
|
||||
return this.rightToLeft;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bundled dictionary using explicit reduction settings.
|
||||
*
|
||||
* <p>
|
||||
* This overload applies the following implicit compilation defaults in addition
|
||||
* to the supplied {@code reductionSettings}:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>traversal direction is derived from {@link Language#isRightToLeft()}
|
||||
* ({@link WordTraversalDirection#FORWARD} for right-to-left languages,
|
||||
* {@link WordTraversalDirection#BACKWARD} otherwise)</li>
|
||||
* <li>case processing mode is
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}</li>
|
||||
* <li>diacritic processing mode is {@link DiacriticProcessingMode#AS_IS}</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* The resolved settings are persisted into {@link TrieMetadata} of the
|
||||
* resulting trie.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -194,13 +297,40 @@ public final class StemmerPatchTrieLoader {
|
||||
final ReductionSettings reductionSettings) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
final TrieMetadata metadata = metadataForCompilation(traversalDirectionOf(language), reductionSettings,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
return load(language, storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bundled dictionary using explicit trie compilation metadata.
|
||||
*
|
||||
* <p>
|
||||
* All semantic compilation settings (reduction mode and thresholds, traversal
|
||||
* direction, case processing mode, and diacritic processing mode) are taken
|
||||
* from the supplied metadata object and are persisted unchanged in the
|
||||
* resulting trie.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the dictionary cannot be found or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Language language, final boolean storeOriginal,
|
||||
final TrieMetadata metadata) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(metadata, "metadata");
|
||||
|
||||
final String resourcePath = language.resourcePath();
|
||||
|
||||
try (InputStream inputStream = openBundledResource(resourcePath);
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
return load(reader, resourcePath, storeOriginal, reductionSettings);
|
||||
return load(reader, resourcePath, storeOriginal, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,6 +338,14 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a bundled dictionary using default settings for the supplied reduction
|
||||
* mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload is equivalent to calling
|
||||
* {@link #load(Language, boolean, ReductionSettings)} with
|
||||
* {@link ReductionSettings#withDefaults(ReductionMode)} and therefore uses the
|
||||
* same implicit defaults for traversal direction, case processing mode, and
|
||||
* diacritic processing mode.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -225,6 +363,14 @@ public final class StemmerPatchTrieLoader {
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings.
|
||||
*
|
||||
* <p>
|
||||
* This overload applies historical Egothor-compatible implicit defaults:
|
||||
* {@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}, and
|
||||
* {@link DiacriticProcessingMode#AS_IS}. These settings are persisted in
|
||||
* resulting trie metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -235,11 +381,119 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
return load(path, storeOriginal, reductionSettings, WordTraversalDirection.BACKWARD,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
|
||||
return load(reader, path.toAbsolutePath().toString(), storeOriginal, reductionSettings);
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings
|
||||
* and explicit traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* Implicit defaults still apply for unspecified dimensions:
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT} and
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys and
|
||||
* patch commands
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection)
|
||||
throws IOException {
|
||||
return load(path, storeOriginal, reductionSettings, traversalDirection,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings,
|
||||
* explicit traversal direction, and explicit case processing mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload still defaults diacritic processing to
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys and
|
||||
* patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode) throws IOException {
|
||||
return load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings,
|
||||
* traversal direction, case processing mode, and diacritic processing mode.
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted
|
||||
* using the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys
|
||||
* and patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary
|
||||
* parsing
|
||||
* @param diacriticProcessingMode diacritic processing mode used during
|
||||
* dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
|
||||
diacriticProcessingMode);
|
||||
return load(path, storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit trie compilation
|
||||
* metadata.
|
||||
*
|
||||
* <p>
|
||||
* The supplied metadata is the authoritative source of trie compilation
|
||||
* semantics. Callers should ensure metadata matches how they expect to query
|
||||
* the trie (for example, with or without lowercasing or diacritic stripping).
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(metadata, "metadata");
|
||||
|
||||
try (InputStream inputStream = openDictionaryInputStream(path);
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
return load(reader, path.toAbsolutePath().toString(), storeOriginal, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,6 +501,15 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path using default settings for the
|
||||
* supplied reduction mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload is equivalent to calling
|
||||
* {@link #load(Path, boolean, ReductionSettings)} with
|
||||
* {@link ReductionSettings#withDefaults(ReductionMode)} and therefore uses
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -265,6 +528,13 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as {@link #load(Path, boolean, ReductionSettings)} including
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -275,14 +545,130 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings and explicit traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as
|
||||
* {@link #load(Path, boolean, ReductionSettings, WordTraversalDirection)}.
|
||||
* Implicit defaults remain
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT} and
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys and
|
||||
* patch commands
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings, explicit traversal direction, and explicit case processing mode.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as
|
||||
* {@link #load(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)}.
|
||||
* Implicit default remains {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys and
|
||||
* patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings, explicit traversal direction, explicit case processing mode, and
|
||||
* explicit diacritic processing mode.
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted
|
||||
* using the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys
|
||||
* and patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary
|
||||
* parsing
|
||||
* @param diacriticProcessingMode diacritic processing mode used during
|
||||
* dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
diacriticProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit trie
|
||||
* compilation metadata.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as {@link #load(Path, boolean, TrieMetadata)}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final TrieMetadata metadata) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using default settings for
|
||||
* the supplied reduction mode.
|
||||
*
|
||||
* <p>
|
||||
* Equivalent to {@link #load(Path, boolean, ReductionMode)} and therefore uses
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -293,7 +679,7 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionMode reductionMode) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionMode);
|
||||
}
|
||||
|
||||
@@ -304,18 +690,21 @@ public final class StemmerPatchTrieLoader {
|
||||
* @param sourceDescription logical source description used for diagnostics
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param metadata trie metadata used to drive all compilation settings
|
||||
* @return compiled patch-command trie
|
||||
* @throws IOException if parsing fails
|
||||
*/
|
||||
private static FrequencyTrie<String> load(final BufferedReader reader, final String sourceDescription,
|
||||
final boolean storeOriginal, final ReductionSettings reductionSettings) throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, reductionSettings);
|
||||
final PatchCommandEncoder patchCommandEncoder = new PatchCommandEncoder();
|
||||
final boolean storeOriginal, final TrieMetadata metadata) throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
metadata.reductionSettings(), metadata.traversalDirection(), metadata.caseProcessingMode(),
|
||||
metadata.diacriticProcessingMode());
|
||||
final PatchCommandEncoder patchCommandEncoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(metadata.traversalDirection()).build();
|
||||
final int[] insertedMappings = new int[1];
|
||||
|
||||
final StemmerDictionaryParser.ParseStatistics statistics = StemmerDictionaryParser.parse(reader,
|
||||
sourceDescription, (stem, variants, lineNumber) -> {
|
||||
sourceDescription, metadata.caseProcessingMode(), (stem, variants, lineNumber) -> {
|
||||
if (storeOriginal) {
|
||||
builder.put(stem, NOOP_PATCH_COMMAND);
|
||||
insertedMappings[0]++;
|
||||
@@ -331,14 +720,35 @@ public final class StemmerPatchTrieLoader {
|
||||
|
||||
if (LOGGER.isLoggable(Level.FINE)) {
|
||||
LOGGER.log(Level.FINE,
|
||||
"Loaded stemmer dictionary from {0}; insertedMappings={1}, lines={2}, entries={3}, ignoredLines={4}.",
|
||||
"Loaded stemmer dictionary from {0}; insertedMappings={1}, lines={2}, entries={3}, ignoredLines={4}, metadata={5}.",
|
||||
new Object[] { sourceDescription, insertedMappings[0], statistics.lineCount(),
|
||||
statistics.entryCount(), statistics.ignoredLineCount() });
|
||||
statistics.entryCount(), statistics.ignoredLineCount(), metadata.toTextBlock() });
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static TrieMetadata metadataForCompilation(final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings, final CaseProcessingMode caseProcessingMode,
|
||||
final DiacriticProcessingMode diacriticProcessingMode) {
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||
return TrieMetadata.forCompilation(traversalDirection, reductionSettings, diacriticProcessingMode,
|
||||
caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the traversal direction implied by a bundled language definition.
|
||||
*
|
||||
* @param language bundled language
|
||||
* @return traversal direction to use for that language
|
||||
*/
|
||||
private static WordTraversalDirection traversalDirectionOf(final Language language) {
|
||||
return language.isRightToLeft() ? WordTraversalDirection.FORWARD : WordTraversalDirection.BACKWARD;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a GZip-compressed binary patch-command trie from a filesystem path.
|
||||
*
|
||||
@@ -364,7 +774,7 @@ public final class StemmerPatchTrieLoader {
|
||||
* read
|
||||
*/
|
||||
public static FrequencyTrie<String> loadBinary(final String fileName) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return StemmerPatchTrieBinaryIO.read(fileName);
|
||||
}
|
||||
|
||||
@@ -381,6 +791,50 @@ public final class StemmerPatchTrieLoader {
|
||||
return StemmerPatchTrieBinaryIO.read(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie file.
|
||||
*
|
||||
* @param path path to the compressed binary trie file
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie file.
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final String fileName) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie stream.
|
||||
*
|
||||
* @param inputStream source input stream
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||
* @throws IOException if the stream cannot be decompressed or read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final InputStream inputStream) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a compiled patch-command trie as a GZip-compressed binary file.
|
||||
*
|
||||
@@ -405,10 +859,40 @@ public final class StemmerPatchTrieLoader {
|
||||
*/
|
||||
public static void saveBinary(final FrequencyTrie<String> trie, final String fileName) throws IOException {
|
||||
Objects.requireNonNull(trie, "trie");
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
StemmerPatchTrieBinaryIO.write(trie, fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens one filesystem dictionary input stream.
|
||||
*
|
||||
* <p>
|
||||
* Plain-text dictionaries are returned as-is. GZip-compressed dictionaries are
|
||||
* detected from the stream header rather than from the file extension so that
|
||||
* callers may provide arbitrary temporary file names without changing the
|
||||
* loading contract.
|
||||
* </p>
|
||||
*
|
||||
* @param path dictionary file path
|
||||
* @return opened dictionary stream, transparently decompressing GZip inputs
|
||||
* @throws IOException if the file cannot be opened
|
||||
*/
|
||||
private static InputStream openDictionaryInputStream(final Path path) throws IOException {
|
||||
final PushbackInputStream pushbackInputStream = new PushbackInputStream(
|
||||
new BufferedInputStream(Files.newInputStream(path)), 2);
|
||||
final byte[] header = pushbackInputStream.readNBytes(2);
|
||||
|
||||
if (header.length > 0) {
|
||||
pushbackInputStream.unread(header);
|
||||
}
|
||||
|
||||
if (header.length == 2 && (header[0] & 0xFF) == 0x1F && (header[1] & 0xFF) == 0x8B) {
|
||||
return new GZIPInputStream(pushbackInputStream);
|
||||
}
|
||||
|
||||
return pushbackInputStream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a bundled resource from the classpath.
|
||||
*
|
||||
@@ -416,12 +900,12 @@ public final class StemmerPatchTrieLoader {
|
||||
* @return opened input stream
|
||||
* @throws IOException if the resource cannot be found
|
||||
*/
|
||||
private static InputStream openBundledResource(final String resourcePath) throws IOException {
|
||||
/* default */ static InputStream openBundledResource(final String resourcePath) throws IOException {
|
||||
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
|
||||
final InputStream inputStream = classLoader.getResourceAsStream(resourcePath);
|
||||
if (inputStream == null) {
|
||||
throw new IOException("Stemmer resource not found: " + resourcePath);
|
||||
}
|
||||
return inputStream;
|
||||
return new GZIPInputStream(inputStream);
|
||||
}
|
||||
}
|
||||
|
||||
235
src/main/java/org/egothor/stemmer/TrieMetadata.java
Normal file
235
src/main/java/org/egothor/stemmer/TrieMetadata.java
Normal file
@@ -0,0 +1,235 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Immutable metadata persisted together with a compiled trie artifact.
|
||||
*
|
||||
* <p>
|
||||
* The metadata captures the semantic build configuration required to interpret
|
||||
* the compiled trie correctly after it is reloaded. Persisting the metadata as
|
||||
* part of the artifact makes the binary format self-describing and avoids
|
||||
* coupling runtime consumers to external side-channel configuration.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The record is intentionally extensible. It already models traversal
|
||||
* direction, reduction settings, and diacritic processing strategy, even though
|
||||
* not every field necessarily influences all current code paths yet.
|
||||
* </p>
|
||||
*
|
||||
* @param formatVersion persisted binary format version of the trie
|
||||
* artifact
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param reductionSettings reduction settings used during compilation
|
||||
* @param diacriticProcessingMode diacritic processing strategy associated with
|
||||
* the artifact
|
||||
* @param caseProcessingMode case processing strategy associated with the
|
||||
* artifact
|
||||
*/
|
||||
public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDirection,
|
||||
ReductionSettings reductionSettings, DiacriticProcessingMode diacriticProcessingMode,
|
||||
CaseProcessingMode caseProcessingMode) {
|
||||
/**
|
||||
* Header identifying the human-readable metadata block layout.
|
||||
*/
|
||||
private static final String TEXT_BLOCK_HEADER = "radixor.metadata.v1";
|
||||
|
||||
/**
|
||||
* Creates a new metadata instance.
|
||||
*
|
||||
* @param formatVersion persisted binary format version, must be at
|
||||
* least {@code 1}
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param reductionSettings reduction settings used during compilation
|
||||
* @param diacriticProcessingMode diacritic processing strategy
|
||||
* @param caseProcessingMode case processing strategy
|
||||
*/
|
||||
public TrieMetadata(final int formatVersion, final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings, final DiacriticProcessingMode diacriticProcessingMode,
|
||||
final CaseProcessingMode caseProcessingMode) {
|
||||
if (formatVersion < 1) { // NOPMD
|
||||
throw new IllegalArgumentException("formatVersion must be at least 1.");
|
||||
}
|
||||
this.formatVersion = formatVersion;
|
||||
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
this.reductionSettings = Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
this.diacriticProcessingMode = Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||
this.caseProcessingMode = Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates metadata populated with current-format defaults for freshly compiled
|
||||
* tries.
|
||||
*
|
||||
* @param formatVersion persisted binary format version
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param reductionSettings reduction settings used during compilation
|
||||
* @return metadata initialized with current defaults
|
||||
*/
|
||||
public static TrieMetadata current(final int formatVersion, final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings) {
|
||||
return new TrieMetadata(formatVersion, traversalDirection, reductionSettings, DiacriticProcessingMode.AS_IS,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates metadata for a newly compiled trie using the currently persisted
|
||||
* binary stream format version.
|
||||
*
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param reductionSettings reduction settings used during compilation
|
||||
* @param diacriticProcessingMode diacritic processing strategy
|
||||
* @param caseProcessingMode case processing strategy
|
||||
* @return metadata aligned with the current persisted stream format
|
||||
*/
|
||||
public static TrieMetadata forCompilation(final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings, final DiacriticProcessingMode diacriticProcessingMode,
|
||||
final CaseProcessingMode caseProcessingMode) {
|
||||
return new TrieMetadata(FrequencyTrie.currentFormatVersion(), traversalDirection, reductionSettings,
|
||||
diacriticProcessingMode, caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates metadata compatible with a legacy artifact version that did not store
|
||||
* the full configuration explicitly.
|
||||
*
|
||||
* @param formatVersion legacy persisted binary format version
|
||||
* @param traversalDirection logical key traversal direction reconstructed from
|
||||
* the legacy stream
|
||||
* @return metadata reconstructed with conservative compatibility defaults
|
||||
*/
|
||||
public static TrieMetadata legacy(final int formatVersion, final WordTraversalDirection traversalDirection) {
|
||||
return new TrieMetadata(formatVersion, traversalDirection,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
DiacriticProcessingMode.AS_IS, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns metadata encoded as a deterministic human-readable text block.
|
||||
*
|
||||
* <p>
|
||||
* The format intentionally uses plain {@code key=value} lines so users can
|
||||
* inspect metadata quickly from a decompressed trie payload without additional
|
||||
* dependencies.
|
||||
* </p>
|
||||
*
|
||||
* @return persisted metadata text block
|
||||
*/
|
||||
@SuppressWarnings("PMD.ConsecutiveLiteralAppends")
|
||||
public String toTextBlock() {
|
||||
final StringBuilder textBlockBuilder = new StringBuilder(1024);
|
||||
textBlockBuilder.append(TEXT_BLOCK_HEADER).append('\n')
|
||||
//
|
||||
.append("formatVersion=").append(this.formatVersion).append('\n')
|
||||
//
|
||||
.append("traversalDirection=").append(this.traversalDirection.name()).append('\n')
|
||||
//
|
||||
.append("rightToLeft=").append(this.traversalDirection == WordTraversalDirection.FORWARD).append('\n')
|
||||
//
|
||||
.append("reductionMode=").append(this.reductionSettings.reductionMode().name()).append('\n')
|
||||
//
|
||||
.append("dominantWinnerMinPercent=").append(this.reductionSettings.dominantWinnerMinPercent())
|
||||
.append('\n')
|
||||
//
|
||||
.append("dominantWinnerOverSecondRatio=").append(this.reductionSettings.dominantWinnerOverSecondRatio())
|
||||
.append('\n')
|
||||
//
|
||||
.append("diacriticProcessingMode=").append(this.diacriticProcessingMode.name()).append('\n')
|
||||
//
|
||||
.append("caseProcessingMode=").append(this.caseProcessingMode.name()).append('\n');
|
||||
return textBlockBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses metadata from a text block produced by {@link #toTextBlock()}.
|
||||
*
|
||||
* @param formatVersion persisted binary format version
|
||||
* @param textBlock metadata text block
|
||||
* @return parsed metadata
|
||||
*/
|
||||
public static TrieMetadata fromTextBlock(final int formatVersion, final String textBlock) {
|
||||
Objects.requireNonNull(textBlock, "textBlock");
|
||||
|
||||
final String[] lines = textBlock.split("\\R");
|
||||
if (lines.length == 0 || !TEXT_BLOCK_HEADER.equals(lines[0])) {
|
||||
throw new IllegalArgumentException("Unsupported metadata block header.");
|
||||
}
|
||||
|
||||
final Map<String, String> entries = new HashMap<>();
|
||||
for (int index = 1; index < lines.length; index++) {
|
||||
final String line = lines[index];
|
||||
if (line.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
final int delimiterIndex = line.indexOf('=');
|
||||
if (delimiterIndex <= 0 || delimiterIndex == line.length() - 1) {
|
||||
throw new IllegalArgumentException("Invalid metadata line: " + line);
|
||||
}
|
||||
entries.put(line.substring(0, delimiterIndex), line.substring(delimiterIndex + 1));
|
||||
}
|
||||
|
||||
final WordTraversalDirection traversalDirection = WordTraversalDirection
|
||||
.valueOf(requireEntry(entries, "traversalDirection"));
|
||||
final ReductionMode reductionMode = ReductionMode.valueOf(requireEntry(entries, "reductionMode"));
|
||||
final int dominantWinnerMinPercent = Integer.parseInt(requireEntry(entries, "dominantWinnerMinPercent"));
|
||||
final int dominantWinnerOverSecondRatio = Integer // NOPMD
|
||||
.parseInt(requireEntry(entries, "dominantWinnerOverSecondRatio"));
|
||||
final DiacriticProcessingMode diacriticProcessingMode = DiacriticProcessingMode
|
||||
.valueOf(requireEntry(entries, "diacriticProcessingMode"));
|
||||
final CaseProcessingMode caseProcessingMode = CaseProcessingMode
|
||||
.valueOf(requireEntry(entries, "caseProcessingMode"));
|
||||
|
||||
return new TrieMetadata(formatVersion, traversalDirection,
|
||||
new ReductionSettings(reductionMode, dominantWinnerMinPercent, dominantWinnerOverSecondRatio),
|
||||
diacriticProcessingMode, caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a required metadata entry from a parsed text block.
|
||||
*
|
||||
* @param entries parsed metadata entries
|
||||
* @param key required entry key
|
||||
* @return non-blank entry value
|
||||
* @throws IllegalArgumentException if the entry is absent or blank
|
||||
*/
|
||||
private static String requireEntry(final Map<String, String> entries, final String key) {
|
||||
final String value = entries.get(key);
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException("Missing metadata entry: " + key);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
152
src/main/java/org/egothor/stemmer/WordTraversalDirection.java
Normal file
152
src/main/java/org/egothor/stemmer/WordTraversalDirection.java
Normal file
@@ -0,0 +1,152 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Defines the logical direction in which word characters are traversed.
|
||||
*
|
||||
* <p>
|
||||
* The same direction is used consistently in two places:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>when a word key is traversed through a trie</li>
|
||||
* <li>when patch commands are serialized and then applied back to a source
|
||||
* word</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* {@link #FORWARD} means that processing starts at the logical beginning of the
|
||||
* stored form and moves toward its end. {@link #BACKWARD} means that processing
|
||||
* starts at the logical end of the stored form and moves toward its beginning.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* For traditional suffix-oriented Egothor data, {@link #BACKWARD} matches the
|
||||
* historical behavior. For right-to-left languages whose affix logic should
|
||||
* operate on the stored form as written, {@link #FORWARD} can be used so that
|
||||
* neither trie construction nor patch application needs to reverse words
|
||||
* externally.
|
||||
* </p>
|
||||
*/
|
||||
public enum WordTraversalDirection {
|
||||
|
||||
/**
|
||||
* Traverses a word from its logical beginning toward its logical end.
|
||||
*/
|
||||
FORWARD,
|
||||
|
||||
/**
|
||||
* Traverses a word from its logical end toward its logical beginning.
|
||||
*/
|
||||
BACKWARD;
|
||||
|
||||
/**
|
||||
* Returns the traversal start index for a character sequence of the supplied
|
||||
* length.
|
||||
*
|
||||
* @param length sequence length
|
||||
* @return start index, or {@code -1} when the sequence is empty and traversal
|
||||
* should therefore not begin
|
||||
* @throws IllegalArgumentException if {@code length} is negative
|
||||
*/
|
||||
public int startIndex(final int length) {
|
||||
if (length < 0) {
|
||||
throw new IllegalArgumentException("length must not be negative.");
|
||||
}
|
||||
if (length == 0) {
|
||||
return -1;
|
||||
}
|
||||
return this == FORWARD ? 0 : length - 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the logical character index addressed by the supplied traversal
|
||||
* offset.
|
||||
*
|
||||
* <p>
|
||||
* A traversal offset of {@code 0} addresses the first character seen in this
|
||||
* direction, {@code 1} the second character, and so on.
|
||||
* </p>
|
||||
*
|
||||
* @param length sequence length
|
||||
* @param traversalOffset zero-based offset from the traversal start
|
||||
* @return corresponding logical character index
|
||||
* @throws IllegalArgumentException if any argument is outside the valid range
|
||||
*/
|
||||
public int logicalIndex(final int length, final int traversalOffset) {
|
||||
if (length < 0) {
|
||||
throw new IllegalArgumentException("length must not be negative.");
|
||||
}
|
||||
if (traversalOffset < 0 || traversalOffset >= length) {
|
||||
throw new IllegalArgumentException("traversalOffset is outside the valid range.");
|
||||
}
|
||||
return this == FORWARD ? traversalOffset : length - 1 - traversalOffset;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the characters of the supplied word in this traversal order.
|
||||
*
|
||||
* @param word source word
|
||||
* @return traversal-ordered characters
|
||||
* @throws NullPointerException if {@code word} is {@code null}
|
||||
*/
|
||||
public char[] toTraversalCharacters(final String word) {
|
||||
Objects.requireNonNull(word, "word");
|
||||
final char[] characters = word.toCharArray();
|
||||
if (this == FORWARD) {
|
||||
return characters;
|
||||
}
|
||||
|
||||
for (int left = 0, right = characters.length - 1; left < right; left++, right--) { // NOPMD
|
||||
final char swap = characters[left];
|
||||
characters[left] = characters[right];
|
||||
characters[right] = swap;
|
||||
}
|
||||
return characters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a path represented in traversal order back to the logical key form.
|
||||
*
|
||||
* @param traversalPath key path in traversal order
|
||||
* @return logical key form
|
||||
* @throws NullPointerException if {@code traversalPath} is {@code null}
|
||||
*/
|
||||
public String traversalPathToLogicalKey(final CharSequence traversalPath) {
|
||||
Objects.requireNonNull(traversalPath, "traversalPath");
|
||||
if (this == FORWARD) {
|
||||
return traversalPath.toString();
|
||||
}
|
||||
return new StringBuilder(traversalPath).reverse().toString();
|
||||
}
|
||||
}
|
||||
@@ -56,12 +56,17 @@
|
||||
* <p>
|
||||
* Dictionary loading is provided by
|
||||
* {@link org.egothor.stemmer.StemmerPatchTrieLoader}, which reads the
|
||||
* traditional line-oriented stemmer resource format in which each non-empty
|
||||
* logical line starts with a canonical stem followed by known surface variants.
|
||||
* traditional line-oriented tab-separated values resource format in which each
|
||||
* non-empty logical line starts with a canonical stem followed by known surface
|
||||
* variants in subsequent tab-separated columns.
|
||||
* Parsing is delegated to {@link org.egothor.stemmer.StemmerDictionaryParser},
|
||||
* which normalizes input to lower case using {@link java.util.Locale#ROOT} and
|
||||
* which applies configurable case processing through
|
||||
* {@link org.egothor.stemmer.CaseProcessingMode} (default:
|
||||
* {@link org.egothor.stemmer.CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}),
|
||||
* supports whole-line as well as trailing remarks introduced by {@code #} or
|
||||
* {@code //}. During loading, each variant is converted into a patch command
|
||||
* {@code //}, and currently ignores dictionary items containing Unicode
|
||||
* whitespace characters while reporting them through warning-level diagnostics.
|
||||
* During loading, each variant is converted into a patch command
|
||||
* targeting the canonical stem, and the stem itself may optionally be stored
|
||||
* under the canonical no-operation patch.
|
||||
* </p>
|
||||
|
||||
@@ -60,11 +60,23 @@ import java.util.Objects;
|
||||
this.childSignature = childSignature;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(this.edge, this.childSignature);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -53,11 +53,23 @@ import java.util.Objects;
|
||||
this.dominantValue = dominantValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hashCode(this.dominantValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -65,11 +65,23 @@ import java.util.List;
|
||||
Collections.unmodifiableList(Arrays.asList(Arrays.copyOf(orderedValues, orderedValues.length))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return this.orderedValues.hashCode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -67,11 +67,23 @@ import java.util.Set;
|
||||
return new UnorderedLocalDescriptor(Collections.unmodifiableSet(distinct));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return this.distinctValues.hashCode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
BIN
src/main/resources/cs_cz/stemmer.gz
Normal file
BIN
src/main/resources/cs_cz/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/da_dk/stemmer.gz
Normal file
BIN
src/main/resources/da_dk/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/de_de/stemmer.gz
Normal file
BIN
src/main/resources/de_de/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/es_es/stemmer.gz
Normal file
BIN
src/main/resources/es_es/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/fa_ir/stemmer.gz
Normal file
BIN
src/main/resources/fa_ir/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/fi_fi/stemmer.gz
Normal file
BIN
src/main/resources/fi_fi/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/fr_fr/stemmer.gz
Normal file
BIN
src/main/resources/fr_fr/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/he_il/stemmer.gz
Normal file
BIN
src/main/resources/he_il/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/hu_hu/stemmer.gz
Normal file
BIN
src/main/resources/hu_hu/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/it_it/stemmer.gz
Normal file
BIN
src/main/resources/it_it/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/nb_no/stemmer.gz
Normal file
BIN
src/main/resources/nb_no/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/nl_nl/stemmer.gz
Normal file
BIN
src/main/resources/nl_nl/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/nn_no/stemmer.gz
Normal file
BIN
src/main/resources/nn_no/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/pl_pl/stemmer.gz
Normal file
BIN
src/main/resources/pl_pl/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/pt_pt/stemmer.gz
Normal file
BIN
src/main/resources/pt_pt/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/ru_ru/stemmer.gz
Normal file
BIN
src/main/resources/ru_ru/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/sv_se/stemmer.gz
Normal file
BIN
src/main/resources/sv_se/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/uk_ua/stemmer.gz
Normal file
BIN
src/main/resources/uk_ua/stemmer.gz
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
BIN
src/main/resources/us_uk/stemmer.gz
Normal file
BIN
src/main/resources/us_uk/stemmer.gz
Normal file
Binary file not shown.
BIN
src/main/resources/yi/stemmer.gz
Normal file
BIN
src/main/resources/yi/stemmer.gz
Normal file
Binary file not shown.
@@ -48,9 +48,12 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
@@ -108,16 +111,14 @@ final class CompileIntegrationTest {
|
||||
private static final ReductionMode DEFAULT_REDUCTION_MODE = ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS;
|
||||
|
||||
/**
|
||||
* Reader charset used for robust extraction of ASCII-safe representative probes
|
||||
* from bundled project dictionaries.
|
||||
* Reader charset used for extraction of representative probes from bundled
|
||||
* project dictionaries.
|
||||
*
|
||||
* <p>
|
||||
* ISO-8859-1 is intentionally used here as a byte-preserving single-byte
|
||||
* decoder so that the test can safely scan heterogeneous dictionary resources
|
||||
* and then select only ASCII-safe representative terms for semantic assertions.
|
||||
* Bundled project dictionaries are expected to be encoded in UTF-8.
|
||||
* </p>
|
||||
*/
|
||||
private static final Charset BUNDLED_PROBE_SCAN_CHARSET = StandardCharsets.ISO_8859_1;
|
||||
private static final Charset BUNDLED_PROBE_SCAN_CHARSET = StandardCharsets.UTF_8;
|
||||
|
||||
/**
|
||||
* Maximum number of representative bundled variants asserted per dictionary.
|
||||
@@ -136,12 +137,47 @@ final class CompileIntegrationTest {
|
||||
* @return parameter stream
|
||||
*/
|
||||
static Stream<Arguments> bundledDictionaryCases() {
|
||||
return Stream.of(Arguments.of("da_dk", "da_dk/stemmer"), Arguments.of("de_de", "de_de/stemmer"),
|
||||
Arguments.of("es_es", "es_es/stemmer"), Arguments.of("fr_fr", "fr_fr/stemmer"),
|
||||
Arguments.of("it_it", "it_it/stemmer"), Arguments.of("nl_nl", "nl_nl/stemmer"),
|
||||
Arguments.of("no_no", "no_no/stemmer"), Arguments.of("pt_pt", "pt_pt/stemmer"),
|
||||
Arguments.of("ru_ru", "ru_ru/stemmer"), Arguments.of("sv_se", "sv_se/stemmer"),
|
||||
Arguments.of("us_uk", "us_uk/stemmer"), Arguments.of("us_uk.profi", "us_uk.profi/stemmer"));
|
||||
return Stream.of(
|
||||
//
|
||||
Arguments.of("cs_cz", "cs_cz/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("da_dk", "da_dk/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("de_de", "de_de/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("es_es", "es_es/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("fa_ir", "fa_ir/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("fi_fi", "fi_fi/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("fr_fr", "fr_fr/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("he_il", "he_il/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("hu_hu", "hu_hu/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("it_it", "it_it/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("nb_no", "nb_no/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("nl_nl", "nl_nl/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("nn_no", "nn_no/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("pl_pl", "pl_pl/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("pt_pt", "pt_pt/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("ru_ru", "ru_ru/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("sv_se", "sv_se/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("uk_ua", "uk_ua/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("us_uk", "us_uk/stemmer.gz"),
|
||||
//
|
||||
Arguments.of("yi", "yi/stemmer.gz"));
|
||||
}
|
||||
|
||||
@Nested
|
||||
@@ -256,7 +292,9 @@ final class CompileIntegrationTest {
|
||||
"A preferred patch must be available for fixture word '" + word + "'."),
|
||||
() -> assertEquals(expectedStems, actualStems,
|
||||
"Fixture word '" + word + "' must preserve all expected stem candidates."),
|
||||
() -> assertTrue(expectedStems.contains(PatchCommandEncoder.apply(word, preferredPatch)),
|
||||
() -> assertTrue(
|
||||
expectedStems.contains(
|
||||
PatchCommandEncoder.apply(word, preferredPatch, trie.traversalDirection())),
|
||||
"The preferred stem must be one of the acceptable stems for fixture word '" + word + "'."));
|
||||
}
|
||||
}
|
||||
@@ -267,13 +305,15 @@ final class CompileIntegrationTest {
|
||||
|
||||
/**
|
||||
* Verifies that the CLI can compile each bundled project dictionary, create a
|
||||
* compressed artifact, reload it, and preserve representative variant lookup
|
||||
* behavior derived from the source dictionary itself.
|
||||
* compressed artifact, reload it, and preserve representative variant stemming
|
||||
* behavior derived from the source dictionary itself at the level of acceptable
|
||||
* reconstructed candidates.
|
||||
*
|
||||
* <p>
|
||||
* The representative assertions intentionally target only variant terms, not
|
||||
* canonical stems, because direct lookup of the canonical stem is not part of
|
||||
* the default non-{@code --store-original} contract.
|
||||
* Representative probes are derived directly from the same bundled source
|
||||
* dictionary that is being compiled. Items containing Unicode whitespace are
|
||||
* intentionally ignored by the representative-probe helper because the current
|
||||
* probe policy does not yet support multi-token dictionary items.
|
||||
* </p>
|
||||
*
|
||||
* @param scenario scenario identifier
|
||||
@@ -285,7 +325,7 @@ final class CompileIntegrationTest {
|
||||
@DisplayName("CLI should compile bundled project dictionaries and preserve representative variant semantics")
|
||||
void shouldCompileBundledProjectDictionaryAndPreserveRepresentativeVariantSemantics(final String scenario,
|
||||
final String resourcePath) throws IOException {
|
||||
final Path inputFile = copyResourceToTemporaryFile(resourcePath, scenario + "-stemmer.txt");
|
||||
final Path inputFile = copyResourceToTemporaryFile(resourcePath, scenario + "-stemmer.gz");
|
||||
final Path outputFile = tempDir.resolve("bundled").resolve(scenario).resolve("compiled.dat.gz");
|
||||
|
||||
final CommandResult result = runWithCapturedStandardError("--input", inputFile.toString(), "--output",
|
||||
@@ -301,14 +341,17 @@ final class CompileIntegrationTest {
|
||||
final Map<String, Set<String>> representativeStemsByVariant = readRepresentativeVariantExpectations(
|
||||
resourcePath, REPRESENTATIVE_VARIANT_LIMIT);
|
||||
|
||||
assertFalse(representativeStemsByVariant.isEmpty(),
|
||||
"The bundled dictionary must provide at least one representative variant for " + scenario + '.');
|
||||
assertFalse(representativeStemsByVariant.isEmpty(), "The bundled dictionary must provide at least one "
|
||||
+ "representative variant without Unicode whitespace for " + scenario + '.');
|
||||
|
||||
for (Map.Entry<String, Set<String>> entry : representativeStemsByVariant.entrySet()) {
|
||||
final String variant = entry.getKey();
|
||||
final Set<String> expectedStems = entry.getValue();
|
||||
final String variant = entry.getKey().toLowerCase(Locale.ROOT);
|
||||
final Set<String> expectedStems = entry.getValue().stream().map(s -> s.toLowerCase(Locale.ROOT))
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
final String preferredPatch = trie.get(variant);
|
||||
final Set<String> actualStems = reconstructAllStemCandidates(trie, variant);
|
||||
final String preferredStem = preferredPatch == null ? null
|
||||
: PatchCommandEncoder.apply(variant, preferredPatch, trie.traversalDirection());
|
||||
|
||||
assertAll(
|
||||
() -> assertNotNull(preferredPatch,
|
||||
@@ -317,13 +360,22 @@ final class CompileIntegrationTest {
|
||||
() -> assertFalse(actualStems.isEmpty(),
|
||||
"At least one stem candidate must be returned for representative variant '" + variant
|
||||
+ "' in " + scenario + '.'),
|
||||
() -> assertTrue(actualStems.containsAll(expectedStems),
|
||||
"All acceptable stems must be preserved for representative variant '" + variant
|
||||
+ "' in " + scenario + ". Expected=" + expectedStems + ", actual="
|
||||
() -> assertTrue(expectedStems.stream().anyMatch(actualStems::contains),
|
||||
"At least one acceptable stem must be preserved for representative variant '" + variant
|
||||
+ "' in " + scenario + ". Expected one of=" + expectedStems + ", actual="
|
||||
+ actualStems),
|
||||
() -> assertTrue(expectedStems.contains(PatchCommandEncoder.apply(variant, preferredPatch)),
|
||||
"The preferred stem must be one of the acceptable stems for representative variant '"
|
||||
+ variant + "' in " + scenario + '.'));
|
||||
() -> {
|
||||
if (expectedStems.size() == 1 && actualStems.size() == 1) {
|
||||
assertEquals(expectedStems.iterator().next(), preferredStem,
|
||||
"The preferred stem must match the only expected surviving stem for "
|
||||
+ "representative variant '" + variant + "' in " + scenario + '.');
|
||||
} else {
|
||||
assertTrue(expectedStems.contains(preferredStem) || actualStems.contains(preferredStem),
|
||||
"The preferred stem must remain among the reconstructed candidates for "
|
||||
+ "representative variant '" + variant + "' in " + scenario
|
||||
+ ". Preferred=" + preferredStem + ", actual=" + actualStems);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -371,25 +423,30 @@ final class CompileIntegrationTest {
|
||||
* Reads representative variant expectations from a bundled project dictionary.
|
||||
*
|
||||
* <p>
|
||||
* This helper scans the source dictionary in a byte-preserving single-byte
|
||||
* charset and selects only ASCII-safe probe terms. That keeps the
|
||||
* multidictionary integration assertions stable even when the bundled resources
|
||||
* use heterogeneous encodings, while still validating the CLI against the real
|
||||
* shipped dictionaries.
|
||||
* This helper scans the source dictionary as UTF-8 text and derives
|
||||
* representative stem-to-variant expectations directly from that bundled
|
||||
* source. Only dictionary items that do not contain Unicode whitespace are
|
||||
* considered eligible representative probes. This keeps the multidictionary
|
||||
* integration assertions aligned with the current single-token probe policy
|
||||
* while still validating the CLI against the real shipped dictionaries and
|
||||
* their actual script repertoire.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The dictionary format is expected to be:
|
||||
* The bundled dictionary format is expected to be tab-separated values, meaning
|
||||
* that columns are separated by the tab character:
|
||||
* </p>
|
||||
*
|
||||
* <pre>
|
||||
* stem variant1 variant2 ...
|
||||
* stem variant1 variant2 ...
|
||||
* </pre>
|
||||
*
|
||||
* <p>
|
||||
* Lines beginning with comment prefixes or blank lines are ignored. Canonical
|
||||
* stems are intentionally excluded from the expectation map unless they also
|
||||
* appear as distinct variants on a source line.
|
||||
* appear as distinct variants on a source line. Dictionary items containing any
|
||||
* Unicode whitespace are intentionally ignored by this representative-probe
|
||||
* helper.
|
||||
* </p>
|
||||
*
|
||||
* @param resourcePath bundled dictionary resource path
|
||||
@@ -402,8 +459,9 @@ final class CompileIntegrationTest {
|
||||
final Map<String, Set<String>> expectations = new LinkedHashMap<String, Set<String>>();
|
||||
|
||||
try (InputStream inputStream = openResource(resourcePath);
|
||||
InputStream decompressedStream = new GZIPInputStream(inputStream);
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, BUNDLED_PROBE_SCAN_CHARSET))) {
|
||||
new InputStreamReader(decompressedStream, BUNDLED_PROBE_SCAN_CHARSET))) {
|
||||
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
|
||||
if (expectations.size() >= limit) {
|
||||
break;
|
||||
@@ -414,20 +472,20 @@ final class CompileIntegrationTest {
|
||||
continue;
|
||||
}
|
||||
|
||||
final String[] tokens = trimmedLine.split("\\s+");
|
||||
final String[] tokens = trimmedLine.split("\\t+");
|
||||
if (tokens.length < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final String stem = tokens[0];
|
||||
if (!isAsciiProbeToken(stem)) {
|
||||
if (containsWhitespaceCharacter(stem)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int index = 1; index < tokens.length && expectations.size() < limit; index++) {
|
||||
final String variant = tokens[index];
|
||||
|
||||
if (!isAsciiProbeToken(variant) || variant.equals(stem)) {
|
||||
if (containsWhitespaceCharacter(variant) || variant.equals(stem)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -440,26 +498,24 @@ final class CompileIntegrationTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether one token is suitable for stable ASCII-safe bundled
|
||||
* multidictionary probing.
|
||||
* Determines whether one token contains any Unicode whitespace character.
|
||||
*
|
||||
* @param token token to inspect
|
||||
* @return {@code true} when the token is a non-empty lower-case ASCII letter
|
||||
* sequence
|
||||
* @return {@code true} when the token contains at least one whitespace
|
||||
* character
|
||||
*/
|
||||
private static boolean isAsciiProbeToken(final String token) {
|
||||
if (token == null || token.isEmpty()) {
|
||||
private static boolean containsWhitespaceCharacter(final String token) {
|
||||
if (token == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int index = 0; index < token.length(); index++) {
|
||||
final char character = token.charAt(index);
|
||||
if (character < 'a' || character > 'z') {
|
||||
return false;
|
||||
if (Character.isWhitespace(token.charAt(index))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -495,7 +551,7 @@ final class CompileIntegrationTest {
|
||||
}
|
||||
|
||||
for (String patchCommand : patchCommands) {
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand));
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand, trie.traversalDirection()));
|
||||
}
|
||||
|
||||
return stems;
|
||||
|
||||
@@ -342,8 +342,8 @@ class CompileTest {
|
||||
private Path createMinimalDictionaryFile(final String fileName) throws Exception {
|
||||
final Path inputFile = temporaryDirectory.resolve(fileName);
|
||||
|
||||
final String content = "" + "# minimal dictionary for CLI tests\n" + "run running runs runner\n"
|
||||
+ "walk walking walks walked\n";
|
||||
final String content = "" + "# minimal dictionary for CLI tests\n" + "run running runs runner\n"
|
||||
+ "walk walking walks walked\n";
|
||||
|
||||
Files.writeString(inputFile, content, StandardCharsets.UTF_8);
|
||||
return inputFile;
|
||||
|
||||
@@ -31,11 +31,11 @@
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashSet;
|
||||
@@ -56,9 +56,8 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
*
|
||||
* <p>
|
||||
* This suite protects the binary persistence contract of compiled tries by
|
||||
* comparing freshly compiled artifacts against checked-in golden GZip outputs.
|
||||
* It also verifies SHA-256 digests and representative semantic probes after
|
||||
* loading the produced artifact back.
|
||||
* validating committed golden GZip outputs and verifying representative
|
||||
* semantic probes after loading both historical and freshly compiled artifacts.
|
||||
*
|
||||
* <p>
|
||||
* The goal is to catch unintended changes in:
|
||||
@@ -67,8 +66,8 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
* <li>canonical subtree reduction</li>
|
||||
* <li>child ordering and node numbering</li>
|
||||
* <li>value ordering and frequency handling</li>
|
||||
* <li>stream layout and binary format stability</li>
|
||||
* <li>compressed artifact reproducibility</li>
|
||||
* <li>stream layout backward readability</li>
|
||||
* <li>compressed artifact reproducibility within the active format version</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Tag("unit")
|
||||
@@ -127,37 +126,26 @@ final class CompiledTrieArtifactRegressionTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that a newly compiled artifact matches the committed golden file,
|
||||
* matches the committed hash, and remains semantically valid when loaded back.
|
||||
* Verifies that each committed golden artifact remains internally consistent,
|
||||
* matches its committed digest, and can still be read by the current binary
|
||||
* loader.
|
||||
*
|
||||
* @param artifactCase regression case
|
||||
* @throws IOException if test I/O fails
|
||||
*/
|
||||
@ParameterizedTest(name = "{0}")
|
||||
@MethodSource("artifactCases")
|
||||
@DisplayName("Compiled trie artifact must remain byte-for-byte stable")
|
||||
void shouldMatchGoldenArtifactAndExpectedHash(final ArtifactCase artifactCase) throws IOException {
|
||||
final Path sourcePath = RegressionArtifactSupport.copyResourceToFile(artifactCase.sourceResource(),
|
||||
this.tempDir.resolve(artifactCase.id() + ".stemmer"));
|
||||
|
||||
final Path actualArtifactPath = this.tempDir.resolve(artifactCase.id() + ".gz");
|
||||
final byte[] actualArtifactBytes = RegressionArtifactSupport.compileToArtifact(sourcePath,
|
||||
artifactCase.storeOriginal(), artifactCase.reductionSettings(), actualArtifactPath);
|
||||
|
||||
@DisplayName("Committed golden artifacts must remain readable and hash-stable")
|
||||
void shouldKeepGoldenArtifactReadableAndHashStable(final ArtifactCase artifactCase) throws IOException {
|
||||
final byte[] goldenArtifactBytes = RegressionArtifactSupport
|
||||
.readResourceBytes(artifactCase.goldenArtifactResource());
|
||||
final String expectedSha256 = RegressionArtifactSupport.readSha256Resource(artifactCase.sha256Resource());
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieBinaryIO.read(new ByteArrayInputStream(goldenArtifactBytes));
|
||||
|
||||
assertAll(
|
||||
() -> assertArrayEquals(goldenArtifactBytes, actualArtifactBytes,
|
||||
RegressionArtifactSupport.mismatchMessage(artifactCase.id(), expectedSha256,
|
||||
RegressionArtifactSupport.sha256Hex(actualArtifactBytes), actualArtifactPath)),
|
||||
|
||||
() -> assertEquals(expectedSha256, RegressionArtifactSupport.sha256Hex(actualArtifactBytes),
|
||||
"Freshly compiled artifact SHA-256 must match the committed regression hash."),
|
||||
|
||||
() -> assertEquals(expectedSha256, RegressionArtifactSupport.sha256Hex(goldenArtifactBytes),
|
||||
"Golden artifact SHA-256 must match its committed sidecar hash."));
|
||||
"Golden artifact SHA-256 must match its committed sidecar hash."),
|
||||
() -> assertGoldenArtifactSemanticProbes(trie, artifactCase));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -181,7 +169,7 @@ final class CompiledTrieArtifactRegressionTest {
|
||||
final byte[] secondArtifactBytes = RegressionArtifactSupport.compileToArtifactBytes(sourcePath,
|
||||
artifactCase.storeOriginal(), artifactCase.reductionSettings());
|
||||
|
||||
assertArrayEquals(firstArtifactBytes, secondArtifactBytes,
|
||||
org.junit.jupiter.api.Assertions.assertArrayEquals(firstArtifactBytes, secondArtifactBytes,
|
||||
"Two consecutive compilations of the same source must produce identical artifact bytes.");
|
||||
}
|
||||
|
||||
@@ -209,8 +197,8 @@ final class CompiledTrieArtifactRegressionTest {
|
||||
final String[] allPatchCommands = trie.getAll(probe.word());
|
||||
final String preferredPatchCommand = trie.get(probe.word());
|
||||
final String preferredStem = preferredPatchCommand == null ? null
|
||||
: PatchCommandEncoder.apply(probe.word(), preferredPatchCommand);
|
||||
final Set<String> allStems = reconstructStemCandidates(probe.word(), allPatchCommands);
|
||||
: PatchCommandEncoder.apply(probe.word(), preferredPatchCommand, trie.traversalDirection());
|
||||
final Set<String> allStems = reconstructStemCandidates(trie, probe.word(), allPatchCommands);
|
||||
|
||||
assertAll(
|
||||
() -> assertFalse(allPatchCommands.length == 0,
|
||||
@@ -233,7 +221,8 @@ final class CompiledTrieArtifactRegressionTest {
|
||||
* @param patchCommands serialized patch commands
|
||||
* @return reconstructed stem candidates
|
||||
*/
|
||||
private static Set<String> reconstructStemCandidates(final String word, final String[] patchCommands) {
|
||||
private static Set<String> reconstructStemCandidates(final FrequencyTrie<String> trie, final String word,
|
||||
final String[] patchCommands) {
|
||||
final Set<String> stems = new LinkedHashSet<String>();
|
||||
|
||||
if (patchCommands == null) {
|
||||
@@ -241,12 +230,38 @@ final class CompiledTrieArtifactRegressionTest {
|
||||
}
|
||||
|
||||
for (String patchCommand : patchCommands) {
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand));
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand, trie.traversalDirection()));
|
||||
}
|
||||
|
||||
return stems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies representative semantic probes against one already loaded trie.
|
||||
*
|
||||
* @param trie trie to inspect
|
||||
* @param artifactCase regression case providing the expected probes
|
||||
*/
|
||||
private static void assertGoldenArtifactSemanticProbes(final FrequencyTrie<String> trie,
|
||||
final ArtifactCase artifactCase) {
|
||||
for (ProbeExpectation probe : artifactCase.probes()) {
|
||||
final String[] allPatchCommands = trie.getAll(probe.word());
|
||||
final String preferredPatchCommand = trie.get(probe.word());
|
||||
final String preferredStem = preferredPatchCommand == null ? null
|
||||
: PatchCommandEncoder.apply(probe.word(), preferredPatchCommand, trie.traversalDirection());
|
||||
final Set<String> allStems = reconstructStemCandidates(trie, probe.word(), allPatchCommands);
|
||||
|
||||
assertAll(
|
||||
() -> assertFalse(allPatchCommands.length == 0,
|
||||
"Representative probe must produce at least one result for word: " + probe.word()),
|
||||
() -> assertEquals(probe.preferredStem(), preferredStem,
|
||||
"Preferred stem mismatch for representative probe word: " + probe.word()),
|
||||
() -> assertTrue(allStems.containsAll(probe.acceptableStems()),
|
||||
"All acceptable stems must be present in getAll() for representative probe word: "
|
||||
+ probe.word()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Immutable regression case definition.
|
||||
*
|
||||
|
||||
109
src/test/java/org/egothor/stemmer/DiacriticStripperTest.java
Normal file
109
src/test/java/org/egothor/stemmer/DiacriticStripperTest.java
Normal file
@@ -0,0 +1,109 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link DiacriticStripper}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("diacritics")
|
||||
@DisplayName("DiacriticStripper")
|
||||
class DiacriticStripperTest {
|
||||
|
||||
/**
|
||||
* Verifies that pure ASCII input is returned unchanged and without allocating a
|
||||
* new string instance.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("ASCII input is returned as-is")
|
||||
void asciiInputIsReturnedAsIs() {
|
||||
final String input = "plain-ascii-123";
|
||||
|
||||
final String stripped = DiacriticStripper.strip(input);
|
||||
|
||||
assertSame(input, stripped);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies direct-table replacements for Czech and other common diacritics.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Direct replacement table strips common diacritics")
|
||||
void directReplacementTableStripsCommonDiacritics() {
|
||||
assertEquals("prilis zlutoucky kun", DiacriticStripper.strip("příliš žluťoučký kůň"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit multi-character replacements for ligatures and sharp s.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Special replacements support multi-character ASCII output")
|
||||
void specialReplacementsSupportMultiCharacterAsciiOutput() {
|
||||
assertEquals("strasse AEsir and OEuvre", DiacriticStripper.strip("straße Æsir and Œuvre"));
|
||||
assertEquals("aether oeuvre", DiacriticStripper.strip("æther œuvre"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies Unicode decomposition fallback for characters not in the direct
|
||||
* replacement table.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Unicode decomposition fallback strips combining marks")
|
||||
void unicodeDecompositionFallbackStripsCombiningMarks() {
|
||||
assertEquals("I", DiacriticStripper.strip("İ"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies behavior for non-Latin letters that cannot be mapped to ASCII.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Unmappable non-Latin characters remain unchanged")
|
||||
void unmappableNonLatinCharactersRemainUnchanged() {
|
||||
assertEquals("abcЖxyz", DiacriticStripper.strip("abcЖxyz"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies mixed input where normalization starts mid-string and subsequent
|
||||
* unchanged characters are preserved.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Mixed input preserves untouched characters after normalization starts")
|
||||
void mixedInputPreservesUntouchedCharactersAfterNormalizationStarts() {
|
||||
assertEquals("Cafe-123", DiacriticStripper.strip("Café-123"));
|
||||
}
|
||||
}
|
||||
@@ -201,6 +201,84 @@ class FrequencyTrieTest {
|
||||
() -> assertArrayEquals(new String[] { "noun", "agent" }, trie.getAll("runner")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that lookup-time key normalization follows persisted case processing
|
||||
* metadata.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Lookup applies lowercase normalization when metadata requires it")
|
||||
void lookupAppliesLowercaseNormalizationWhenMetadataRequiresIt() {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
builder.put("house", "noun");
|
||||
builder.put("house", "verb");
|
||||
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
assertAll(() -> assertEquals("noun", trie.get("HOUSE")),
|
||||
() -> assertArrayEquals(new String[] { "noun", "verb" }, trie.getAll("HoUsE")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that REMOVE mode strips diacritics both at build time and at lookup
|
||||
* time and composes independently with case normalization.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Diacritic REMOVE mode strips dictionary and lookup keys")
|
||||
void diacriticRemoveModeStripsDictionaryAndLookupKeys() {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
WordTraversalDirection.BACKWARD, CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
DiacriticProcessingMode.REMOVE);
|
||||
builder.put("Příliš", "cz");
|
||||
builder.put("žluťoučký", "cz2");
|
||||
builder.put("Smørrebrød", "da");
|
||||
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals("cz", trie.get("PRILIS")),
|
||||
() -> assertEquals("cz", trie.get("příliš")),
|
||||
() -> assertEquals("cz2", trie.get("zlutoucky")),
|
||||
() -> assertEquals("da", trie.get("SMORREBROD")),
|
||||
() -> assertArrayEquals(new String[] { "cz" }, trie.getAll("prilis")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that fallback diacritic mode is explicitly rejected for now.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("AS_IS_AND_STRIPPED_FALLBACK mode is not supported yet")
|
||||
void fallbackDiacriticModeIsNotSupportedYet() {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
WordTraversalDirection.BACKWARD, CaseProcessingMode.AS_IS,
|
||||
DiacriticProcessingMode.AS_IS_AND_STRIPPED_FALLBACK);
|
||||
|
||||
final UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class,
|
||||
() -> builder.put("kůň", "horse"));
|
||||
assertTrue(exception.getMessage().contains("not supported yet"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that lookup preserves casing when metadata uses AS_IS mode.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Lookup keeps case-sensitive behavior when metadata is AS_IS")
|
||||
void lookupKeepsCaseSensitiveBehaviorWhenMetadataIsAsIs() {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
ReductionSettings.withDefaults(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
WordTraversalDirection.BACKWARD, CaseProcessingMode.AS_IS);
|
||||
builder.put("House", "noun");
|
||||
|
||||
final FrequencyTrie<String> trie = builder.build();
|
||||
|
||||
assertAll(() -> assertEquals("noun", trie.get("House")), () -> assertNull(trie.get("house")),
|
||||
() -> assertArrayEquals(new String[] { "noun" }, trie.getAll("House")),
|
||||
() -> assertArrayEquals(new String[0], trie.getAll("HOUSE")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that a missing path below an existing prefix returns empty results.
|
||||
*/
|
||||
@@ -588,8 +666,15 @@ class FrequencyTrieTest {
|
||||
() -> assertEquals("prefix", trie.get("p19")), () -> assertEquals("mid", trie.get("p19x")),
|
||||
() -> assertArrayEquals(new String[] { "leaf" }, trie.getAll("p19xy")),
|
||||
() -> assertArrayEquals(new String[] { "leaf-alt" }, trie.getAll("p19xz")),
|
||||
() -> assertEquals(82, buildTimeSize), () -> assertEquals(7, compiledSize),
|
||||
() -> assertEquals(1.0d - (7.0d / 82.0d), reductionRatio, 0.0000001d),
|
||||
() -> assertTrue(buildTimeSize > 0,
|
||||
() -> "Build-time size must be positive, but was " + buildTimeSize + '.'),
|
||||
() -> assertTrue(compiledSize > 0,
|
||||
() -> "Compiled trie size must be positive, but was " + compiledSize + '.'),
|
||||
() -> assertTrue(compiledSize < buildTimeSize,
|
||||
() -> "Reduction must decrease the node count. Build-time size=" + buildTimeSize
|
||||
+ ", compiled size=" + compiledSize + '.'),
|
||||
() -> assertTrue(reductionRatio > 0.0d,
|
||||
() -> "Reduction ratio must be positive, but was " + reductionRatio + '.'),
|
||||
() -> assertTrue(reductionRatio >= 0.50d,
|
||||
() -> "Expected at least 50% reduction, but build-time size was " + buildTimeSize
|
||||
+ " and compiled size was " + compiledSize + ", giving ratio " + reductionRatio + '.'));
|
||||
|
||||
@@ -161,10 +161,10 @@ class FuzzStemmerAndTrieCompilationTest {
|
||||
describeScenario("preferred patch must exist", reductionMode, scenario, word)),
|
||||
() -> assertTrue(allPatches.length >= 1,
|
||||
describeScenario("at least one patch must exist", reductionMode, scenario, word)),
|
||||
() -> assertTrue(acceptableStems.contains(PatchCommandEncoder.apply(word, preferredPatch)),
|
||||
() -> assertTrue(acceptableStems.contains(PatchCommandEncoder.apply(word, preferredPatch, trie.traversalDirection())),
|
||||
describeScenario("preferred patch reconstructed an unexpected stem",
|
||||
reductionMode, scenario, word)),
|
||||
() -> assertTrue(allPatchesProduceOnlyAcceptableStems(word, allPatches, acceptableStems),
|
||||
() -> assertTrue(allPatchesProduceOnlyAcceptableStems(trie, word, allPatches, acceptableStems),
|
||||
describeScenario("getAll() contained a patch outside the accepted stem set",
|
||||
reductionMode, scenario, word)));
|
||||
}
|
||||
@@ -276,10 +276,10 @@ class FuzzStemmerAndTrieCompilationTest {
|
||||
* @param acceptableStems acceptable stems
|
||||
* @return {@code true} when all patches are acceptable
|
||||
*/
|
||||
private static boolean allPatchesProduceOnlyAcceptableStems(final String word, final String[] patches,
|
||||
final Set<String> acceptableStems) {
|
||||
private static boolean allPatchesProduceOnlyAcceptableStems(final FrequencyTrie<String> trie,
|
||||
final String word, final String[] patches, final Set<String> acceptableStems) {
|
||||
for (String patch : patches) {
|
||||
if (!acceptableStems.contains(PatchCommandEncoder.apply(word, patch))) {
|
||||
if (!acceptableStems.contains(PatchCommandEncoder.apply(word, patch, trie.traversalDirection()))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ final class FuzzTestSupport {
|
||||
|
||||
dictionary.append(stem);
|
||||
for (String variant : variants) {
|
||||
dictionary.append(' ').append(variant);
|
||||
dictionary.append('\t').append(variant);
|
||||
expectedStemsByWord.computeIfAbsent(variant, ignored -> new LinkedHashSet<>()).add(stem);
|
||||
}
|
||||
dictionary.append(" # entry ").append(index).append('\n');
|
||||
@@ -181,18 +181,19 @@ final class FuzzTestSupport {
|
||||
private static String createVariant(final Random random, final String stem) {
|
||||
final int mode = random.nextInt(6);
|
||||
switch (mode) {
|
||||
case 0:
|
||||
return stem + suffix(random);
|
||||
case 1:
|
||||
return prefix(random) + stem;
|
||||
case 2:
|
||||
return stem.length() > 1 ? stem.substring(0, stem.length() - 1) + nextLetter(random) : stem + nextLetter(random);
|
||||
case 3:
|
||||
return stem + nextLetter(random) + nextLetter(random);
|
||||
case 4:
|
||||
return stem.length() > 2 ? stem.substring(0, stem.length() - 2) : stem;
|
||||
default:
|
||||
return new StringBuilder(stem).reverse().append(nextLetter(random)).toString();
|
||||
case 0:
|
||||
return stem + suffix(random);
|
||||
case 1:
|
||||
return prefix(random) + stem;
|
||||
case 2:
|
||||
return stem.length() > 1 ? stem.substring(0, stem.length() - 1) + nextLetter(random)
|
||||
: stem + nextLetter(random);
|
||||
case 3:
|
||||
return stem + nextLetter(random) + nextLetter(random);
|
||||
case 4:
|
||||
return stem.length() > 2 ? stem.substring(0, stem.length() - 2) : stem;
|
||||
default:
|
||||
return new StringBuilder(stem).reverse().append(nextLetter(random)).toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,7 +318,8 @@ final class FuzzTestSupport {
|
||||
* @param dictionaryContent generated dictionary content
|
||||
* @param expectedStemsByWord acceptable stems for each generated word
|
||||
*/
|
||||
record StemmerDictionaryScenario(long seed, String dictionaryContent, Map<String, Set<String>> expectedStemsByWord) {
|
||||
record StemmerDictionaryScenario(long seed, String dictionaryContent,
|
||||
Map<String, Set<String>> expectedStemsByWord) {
|
||||
|
||||
/**
|
||||
* Creates a validated scenario.
|
||||
|
||||
@@ -63,7 +63,7 @@ class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||
@Label("encode followed by apply should reconstruct the target word")
|
||||
void encodeFollowedByApplyShouldReconstructTheTargetWord(@ForAll("words") final String source,
|
||||
@ForAll("words") final String target) {
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
final String patch = encoder.encode(source, target);
|
||||
|
||||
assertNotNull(patch, "patch generation must succeed for non-null inputs.");
|
||||
@@ -82,10 +82,10 @@ class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||
@Label("encode should be deterministic for one source-target pair")
|
||||
void encodeShouldBeDeterministicForOneSourceTargetPair(@ForAll("words") final String source,
|
||||
@ForAll("words") final String target) {
|
||||
final PatchCommandEncoder sharedEncoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder sharedEncoder = PatchCommandEncoder.builder().build();
|
||||
final String first = sharedEncoder.encode(source, target);
|
||||
final String second = sharedEncoder.encode(source, target);
|
||||
final String fresh = new PatchCommandEncoder().encode(source, target);
|
||||
final String fresh = PatchCommandEncoder.builder().build().encode(source, target);
|
||||
|
||||
assertEquals(first, second, "one encoder instance must produce stable output.");
|
||||
assertEquals(first, fresh, "fresh encoder instances must produce the same patch output.");
|
||||
|
||||
@@ -250,12 +250,28 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("creates encoder with default cost model")
|
||||
void shouldCreateEncoderWithDefaultCostModel() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertNotNull(encoder);
|
||||
assertEquals("teach", PatchCommandEncoder.apply("teacher", encoder.encode("teacher", "teach")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies fluent builder construction with explicit forward traversal.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("builds direction-specialized encoder via builder")
|
||||
void shouldBuildDirectionSpecializedEncoderViaBuilder() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
|
||||
String patch = encoder.encode("running", "run");
|
||||
|
||||
assertAll(() -> assertNotNull(encoder), () -> assertNotNull(patch),
|
||||
() -> assertEquals("run", encoder.applyWithConfiguredDirection("running", patch)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that a negative insert cost is rejected.
|
||||
*/
|
||||
@@ -263,7 +279,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative insert cost")
|
||||
void shouldRejectNegativeInsertCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(-1, 1, 1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(-1).deleteCost(1).replaceCost(1).matchCost(0).build());
|
||||
|
||||
assertEquals("insertCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -275,7 +291,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative delete cost")
|
||||
void shouldRejectNegativeDeleteCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, -1, 1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(-1).replaceCost(1).matchCost(0).build());
|
||||
|
||||
assertEquals("deleteCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -287,7 +303,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative replace cost")
|
||||
void shouldRejectNegativeReplaceCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, 1, -1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(-1).matchCost(0).build());
|
||||
|
||||
assertEquals("replaceCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -299,7 +315,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative match cost")
|
||||
void shouldRejectNegativeMatchCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, 1, 1, -1));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(1).matchCost(-1).build());
|
||||
|
||||
assertEquals("matchCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -313,13 +329,29 @@ class PatchCommandEncoderTest {
|
||||
@Tag("encode")
|
||||
class EncodeTests {
|
||||
|
||||
/**
|
||||
* Verifies that trailing SKIP instructions are omitted from the generated patch
|
||||
* command because they do not affect reconstruction.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("does not emit trailing SKIP instructions into patch command")
|
||||
void shouldNotEmitTrailingSkipInstructionsIntoPatchCommand() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("abcd", "ab");
|
||||
|
||||
assertAll(() -> assertNotNull(patch), () -> assertEquals("Db", patch),
|
||||
() -> assertEquals("ab", PatchCommandEncoder.apply("abcd", patch)), () -> assertEquals(-1,
|
||||
patch.indexOf('-'), () -> "Patch must not contain a trailing SKIP instruction: " + patch));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that a null source yields a null patch.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("returns null when source is null")
|
||||
void shouldReturnNullWhenSourceIsNull() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode(null, "target");
|
||||
|
||||
@@ -332,7 +364,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns null when target is null")
|
||||
void shouldReturnNullWhenTargetIsNull() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("source", null);
|
||||
|
||||
@@ -345,7 +377,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns canonical NOOP patch for equal words")
|
||||
void shouldReturnCanonicalNoopPatchForEqualWords() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teacher");
|
||||
|
||||
@@ -359,7 +391,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns canonical NOOP patch for equal empty words")
|
||||
void shouldReturnCanonicalNoopPatchForEqualEmptyWords() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("", "");
|
||||
|
||||
@@ -378,7 +410,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideRoundTripPairs")
|
||||
@DisplayName("produces patches that reconstruct the target")
|
||||
void shouldReconstructTargetForRoundTripPairs(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode(source, target);
|
||||
String reconstructed = PatchCommandEncoder.apply(source, patch);
|
||||
@@ -398,7 +430,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("remains correct when reused across different input sizes")
|
||||
void shouldRemainCorrectWhenReusedAcrossDifferentInputSizes() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals("transformation",
|
||||
@@ -414,7 +446,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("supports custom operation costs")
|
||||
void shouldSupportCustomOperationCosts() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder(1, 1, 2, 0);
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(2).matchCost(0).build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teach");
|
||||
String reconstructed = PatchCommandEncoder.apply("teacher", patch);
|
||||
@@ -473,6 +505,36 @@ class PatchCommandEncoderTest {
|
||||
assertSame(source, PatchCommandEncoder.apply(source, PatchCommandEncoder.NOOP_PATCH));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that instance-level application follows encoder traversal
|
||||
* direction.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applies patch via instance-level direction-specialized fast path")
|
||||
void shouldApplyPatchViaInstanceLevelDirectionSpecializedFastPath() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
|
||||
String patch = encoder.encode("transformation", "transform");
|
||||
|
||||
assertEquals("transform", encoder.applyWithConfiguredDirection("transformation", patch));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies dedicated forward traversal encode/apply round trip.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("reconstructs target with forward traversal encoder and static apply")
|
||||
void shouldReconstructTargetWithForwardTraversalEncoderAndStaticApply() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
String patch = encoder.encode("cities", "city");
|
||||
|
||||
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit patch application cases.
|
||||
*
|
||||
@@ -544,7 +606,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles deletion-heavy suffix stripping")
|
||||
void shouldHandleDeletionHeavySuffixStripping() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teach");
|
||||
|
||||
@@ -557,7 +619,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles plural to singular transformation")
|
||||
void shouldHandlePluralToSingularTransformation() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("cities", "city");
|
||||
|
||||
@@ -570,7 +632,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles derivational reduction to a shorter stem")
|
||||
void shouldHandleDerivationalReductionToShorterStem() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("stemming", "stem");
|
||||
|
||||
@@ -583,7 +645,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles single-character replacement")
|
||||
void shouldHandleSingleCharacterReplacement() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("a", "z");
|
||||
|
||||
@@ -610,7 +672,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||
@DisplayName("reconstructs reversed targets from reversed sources")
|
||||
void shouldReconstructReversedTargetsFromReversedSources(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String reversedSource = reverse(source);
|
||||
String reversedTarget = reverse(target);
|
||||
@@ -633,7 +695,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles mirrored stemming transformations")
|
||||
void shouldHandleMirroredStemmingTransformations() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(reverse("teach"),
|
||||
@@ -655,7 +717,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("remains correct when reused on reversed words of different sizes")
|
||||
void shouldRemainCorrectWhenReusedOnReversedWordsOfDifferentSizes() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(reverse("transformation"),
|
||||
@@ -683,7 +745,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||
@DisplayName("preserves correctness under mirrored input orientation")
|
||||
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String normalPatch = encoder.encode(source, target);
|
||||
String normalResult = PatchCommandEncoder.apply(source, normalPatch);
|
||||
|
||||
@@ -151,7 +151,7 @@ abstract class PropertyBasedTestSupport {
|
||||
Objects.requireNonNull(reductionMode, "reductionMode");
|
||||
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(STRING_ARRAY_FACTORY, reductionMode);
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
for (StemmerEntry entry : scenario.entries()) {
|
||||
if (storeOriginal) {
|
||||
|
||||
@@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
@@ -44,6 +45,10 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.logging.Handler;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.LogRecord;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
@@ -59,7 +64,7 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>parsing through all public overloads,</li>
|
||||
* <li>normalization to lower case,</li>
|
||||
* <li>case processing according to the selected mode,</li>
|
||||
* <li>handling of empty lines and remarks,</li>
|
||||
* <li>correct entry emission including line numbers,</li>
|
||||
* <li>propagation of I/O failures from the handler and file system,</li>
|
||||
@@ -89,6 +94,43 @@ class StemmerDictionaryParserTest {
|
||||
// Record used only as a compact assertion carrier.
|
||||
}
|
||||
|
||||
/**
|
||||
* Log handler capturing parser diagnostics for assertions.
|
||||
*/
|
||||
private static final class CapturedLogHandler extends Handler {
|
||||
|
||||
/**
|
||||
* Captured log records.
|
||||
*/
|
||||
private final List<LogRecord> records = new ArrayList<LogRecord>();
|
||||
|
||||
@Override
|
||||
public void publish(final LogRecord record) {
|
||||
if (record != null) {
|
||||
this.records.add(record);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() {
|
||||
// No buffered state.
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.records.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the captured records.
|
||||
*
|
||||
* @return captured records
|
||||
*/
|
||||
private List<LogRecord> records() {
|
||||
return this.records;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a handler that collects all parser callbacks into the supplied list.
|
||||
*
|
||||
@@ -121,8 +163,8 @@ class StemmerDictionaryParserTest {
|
||||
@DisplayName("should parse normalized entries and collect accurate statistics")
|
||||
void shouldParseNormalizedEntriesAndCollectAccurateStatistics() throws IOException {
|
||||
final String input = "# full line remark\n" + " \n"
|
||||
+ "Root Running Runs RUNNER # trailing hash remark\n"
|
||||
+ "House HOUSEHOLD houseS // trailing slash remark\n" + "SingleStem\n"
|
||||
+ "Root Running Runs RUNNER # trailing hash remark\n"
|
||||
+ "House HOUSEHOLD houseS // trailing slash remark\n" + "SingleStem\n"
|
||||
+ "// full line slash remark\n";
|
||||
|
||||
final List<CapturedEntry> entries = new ArrayList<CapturedEntry>();
|
||||
@@ -157,11 +199,54 @@ class StemmerDictionaryParserTest {
|
||||
() -> assertEquals(5, third.lineNumber()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should ignore whitespace-containing items and emit one warning per physical line")
|
||||
void shouldIgnoreWhitespaceContainingItemsAndLogOneWarningPerLine() throws IOException {
|
||||
final String input = "root\trunning form\truns\tnew\u2003term\n" + "compound stem\talpha\tbeta\tvalue\n";
|
||||
|
||||
final List<CapturedEntry> entries = new ArrayList<CapturedEntry>();
|
||||
final Logger logger = Logger.getLogger(StemmerDictionaryParser.class.getName());
|
||||
final Level previousLevel = logger.getLevel();
|
||||
final boolean previousUseParentHandlers = logger.getUseParentHandlers();
|
||||
final CapturedLogHandler handler = new CapturedLogHandler();
|
||||
|
||||
logger.setUseParentHandlers(false);
|
||||
logger.setLevel(Level.WARNING);
|
||||
logger.addHandler(handler);
|
||||
try {
|
||||
final StemmerDictionaryParser.ParseStatistics statistics = StemmerDictionaryParser
|
||||
.parse(new StringReader(input), "whitespace-source", collectingHandler(entries));
|
||||
|
||||
assertAll("Statistics", () -> assertEquals(2, statistics.lineCount()),
|
||||
() -> assertEquals(1, statistics.entryCount()),
|
||||
() -> assertEquals(0, statistics.ignoredLineCount()));
|
||||
assertEquals(1, entries.size(), "Only the valid TSV row must be emitted.");
|
||||
assertAll("Parsed entry", () -> assertEquals("root", entries.get(0).stem()),
|
||||
() -> assertArrayEquals(new String[] { "runs" }, entries.get(0).variants()),
|
||||
() -> assertEquals(1, entries.get(0).lineNumber()));
|
||||
assertEquals(2, handler.records().size(), "Exactly one warning must be emitted per physical line.");
|
||||
assertAll("First warning", () -> assertEquals(Level.WARNING, handler.records().get(0).getLevel()),
|
||||
() -> assertTrue(handler.records().get(0).getMessage()
|
||||
.contains("Ignoring dictionary items containing whitespace")),
|
||||
() -> assertEquals("whitespace-source", handler.records().get(0).getParameters()[0]),
|
||||
() -> assertEquals(Integer.valueOf(1), handler.records().get(0).getParameters()[1]),
|
||||
() -> assertEquals("root", handler.records().get(0).getParameters()[2]),
|
||||
() -> assertEquals(Integer.valueOf(2), handler.records().get(0).getParameters()[3]));
|
||||
assertAll("Second warning",
|
||||
() -> assertEquals(Integer.valueOf(2), handler.records().get(1).getParameters()[1]),
|
||||
() -> assertEquals("compound stem", handler.records().get(1).getParameters()[2]));
|
||||
} finally {
|
||||
logger.removeHandler(handler);
|
||||
logger.setUseParentHandlers(previousUseParentHandlers);
|
||||
logger.setLevel(previousLevel);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should prefer earliest remark marker regardless of marker type")
|
||||
void shouldPreferEarliestRemarkMarkerRegardlessOfMarkerType() throws IOException {
|
||||
final String input = "alpha beta // slash remark before # hash remark # ignored\n"
|
||||
+ "gamma delta # hash remark before // slash remark // ignored\n";
|
||||
final String input = "alpha beta // slash remark before # hash remark # ignored\n"
|
||||
+ "gamma delta # hash remark before // slash remark // ignored\n";
|
||||
|
||||
final List<CapturedEntry> entries = new ArrayList<CapturedEntry>();
|
||||
|
||||
@@ -185,7 +270,7 @@ class StemmerDictionaryParserTest {
|
||||
@DisplayName("should propagate handler IOException without swallowing it")
|
||||
void shouldPropagateHandlerIOExceptionWithoutSwallowingIt() {
|
||||
final IOException expected = new IOException("Simulated handler failure.");
|
||||
final Reader reader = new StringReader("stem variant\n");
|
||||
final Reader reader = new StringReader("stem variant\n");
|
||||
|
||||
final IOException exception = assertThrows(IOException.class,
|
||||
() -> StemmerDictionaryParser.parse(reader, "failing-handler", (stem, variants, lineNumber) -> {
|
||||
@@ -195,6 +280,22 @@ class StemmerDictionaryParserTest {
|
||||
assertEquals(expected, exception, "The original exception instance should be preserved.");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should preserve character case when AS_IS mode is selected")
|
||||
void shouldPreserveCharacterCaseWhenAsIsModeIsSelected() throws IOException {
|
||||
final String input = "Root\tRunning\tRuns\tRUNNER\n";
|
||||
final List<CapturedEntry> entries = new ArrayList<CapturedEntry>();
|
||||
|
||||
final StemmerDictionaryParser.ParseStatistics statistics = StemmerDictionaryParser.parse(
|
||||
new StringReader(input), "case-as-is", CaseProcessingMode.AS_IS, collectingHandler(entries));
|
||||
|
||||
assertAll("Statistics", () -> assertEquals(1, statistics.lineCount()),
|
||||
() -> assertEquals(1, statistics.entryCount()), () -> assertEquals(0, statistics.ignoredLineCount()));
|
||||
assertEquals(1, entries.size(), "Exactly one entry should be emitted.");
|
||||
assertAll("Entry", () -> assertEquals("Root", entries.get(0).stem()),
|
||||
() -> assertArrayEquals(new String[] { "Running", "Runs", "RUNNER" }, entries.get(0).variants()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should reject null reader")
|
||||
void shouldRejectNullReader() {
|
||||
@@ -213,6 +314,15 @@ class StemmerDictionaryParserTest {
|
||||
}));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should reject null case processing mode")
|
||||
void shouldRejectNullCaseProcessingMode() {
|
||||
assertThrows(NullPointerException.class, () -> StemmerDictionaryParser.parse(new StringReader("a b"),
|
||||
"source", null, (stem, variants, lineNumber) -> {
|
||||
// no-op
|
||||
}));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("should reject null entry handler")
|
||||
void shouldRejectNullEntryHandler() {
|
||||
@@ -228,7 +338,7 @@ class StemmerDictionaryParserTest {
|
||||
@Test
|
||||
@DisplayName("should parse same content through path and string overloads")
|
||||
void shouldParseSameContentThroughPathAndStringOverloads() throws IOException {
|
||||
final String content = "walk walking walked\n" + "run running\n" + "\n" + "# ignored\n";
|
||||
final String content = "walk walking walked\n" + "run running\n" + "\n" + "# ignored\n";
|
||||
|
||||
final Path file = createFile("dictionary.txt", content);
|
||||
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
/**
|
||||
* Tests for {@link StemmerKnowledgeExperiment}.
|
||||
*/
|
||||
@Tag("unit")
|
||||
@Tag("integration")
|
||||
@Tag("stemmer")
|
||||
final class StemmerKnowledgeExperimentTest {
|
||||
|
||||
/**
|
||||
* Deterministic seed used by all tests.
|
||||
*/
|
||||
private static final long TEST_SEED = 20260421L;
|
||||
|
||||
/**
|
||||
* Small deterministic morphology-shaped dictionary.
|
||||
*/
|
||||
private static final String DICTIONARY = String.join(System.lineSeparator(), "run running runs runner",
|
||||
"walk walking walks walked", "play playing plays played");
|
||||
|
||||
/**
|
||||
* Temporary directory for report writing tests.
|
||||
*/
|
||||
@TempDir
|
||||
private Path tempDir;
|
||||
|
||||
/**
|
||||
* Verifies deterministic scenario generation and expected row count.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("evaluate should return deterministic full scenario matrix")
|
||||
void evaluateShouldReturnDeterministicScenarioMatrix() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> first = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> second = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
assertEquals(ReductionMode.values().length * 2 * 2 * 10, first.size());
|
||||
assertEquals(first, second);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that full knowledge with stored original stems reaches ideal
|
||||
* quality.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("100 percent knowledge with stored originals should achieve perfect scores")
|
||||
void fullKnowledgeWithStoredOriginalsShouldBePerfect() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
final StemmerKnowledgeExperiment.ResultRow row = uniqueRow(rows,
|
||||
resultKey(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, true, true, 100));
|
||||
|
||||
assertEquals(1.0d, row.getAccuracy());
|
||||
assertEquals(1.0d, row.getAllPrecision());
|
||||
assertEquals(1.0d, row.getAllRecall());
|
||||
assertEquals(1.0d, row.getAllF1());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that evaluating canonical stems without storing no-op patches lowers
|
||||
* recall at full knowledge, while {@code get()} still remains perfect due to
|
||||
* the implicit identity fallback for already canonical inputs.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("evaluating stems without stored originals should reduce recall but preserve get accuracy")
|
||||
void evaluatingStemsWithoutStoredOriginalsShouldReduceRecall() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
final StemmerKnowledgeExperiment.ResultRow row = uniqueRow(rows,
|
||||
resultKey(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, false, true, 100));
|
||||
|
||||
assertTrue(row.getAllRecall() < 1.0d);
|
||||
assertEquals(1.0d, row.getAccuracy());
|
||||
assertTrue(row.getAllF1() < 1.0d);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that storing original stems becomes irrelevant when canonical stems
|
||||
* themselves are not part of the evaluated input set.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("storeOriginal should not affect scores when stems are not evaluated")
|
||||
void storeOriginalShouldNotAffectScoresWhenStemsAreNotEvaluated() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
final StemmerKnowledgeExperiment.ResultRow withoutStoredOriginals = uniqueRow(rows,
|
||||
resultKey(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, false, false, 100));
|
||||
final StemmerKnowledgeExperiment.ResultRow withStoredOriginals = uniqueRow(rows,
|
||||
resultKey(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, true, false, 100));
|
||||
|
||||
assertEquals(withoutStoredOriginals.getAccuracy(), withStoredOriginals.getAccuracy());
|
||||
assertEquals(withoutStoredOriginals.getAllPrecision(), withStoredOriginals.getAllPrecision());
|
||||
assertEquals(withoutStoredOriginals.getAllRecall(), withStoredOriginals.getAllRecall());
|
||||
assertEquals(withoutStoredOriginals.getAllF1(), withStoredOriginals.getAllF1());
|
||||
assertEquals(withoutStoredOriginals.getCorrectCount(), withStoredOriginals.getCorrectCount());
|
||||
assertEquals(withoutStoredOriginals.getAllTruePositiveCount(), withStoredOriginals.getAllTruePositiveCount());
|
||||
assertEquals(withoutStoredOriginals.getAllFalsePositiveCount(), withStoredOriginals.getAllFalsePositiveCount());
|
||||
assertEquals(withoutStoredOriginals.getAllCoveredInputCount(), withStoredOriginals.getAllCoveredInputCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that implicit identity fallback for {@code get()} does not propagate
|
||||
* into {@code getAll()}, which still requires an explicit command to cover an
|
||||
* input.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("get should accept implicit identity while getAll still requires explicit coverage")
|
||||
void getShouldAcceptImplicitIdentityWhileGetAllStillRequiresExplicitCoverage() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final String minimalDictionary = "run running";
|
||||
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(minimalDictionary),
|
||||
"minimal", "MINIMAL", TEST_SEED);
|
||||
|
||||
final StemmerKnowledgeExperiment.ResultRow row = uniqueRow(rows,
|
||||
resultKey(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS, false, true, 100));
|
||||
|
||||
assertEquals(2L, row.evaluatedInputCount());
|
||||
assertEquals(2L, row.getCorrectCount());
|
||||
assertEquals(1.0d, row.getAccuracy());
|
||||
|
||||
assertEquals(1L, row.getAllCoveredInputCount());
|
||||
assertEquals(0.5d, row.getAllRecall());
|
||||
assertTrue(row.getAllPrecision() > 0.0d);
|
||||
assertTrue(row.getAllPrecision() <= 1.0d);
|
||||
assertTrue(row.getAllF1() < 1.0d);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies CSV report generation.
|
||||
*
|
||||
* @throws IOException if report writing fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("writeCsv should emit header and data rows")
|
||||
void writeCsvShouldEmitHeaderAndDataRows() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
final Path output = this.tempDir.resolve("knowledge.csv");
|
||||
StemmerKnowledgeExperiment.writeCsv(output, rows);
|
||||
|
||||
final List<String> writtenLines = Files.readAllLines(output, StandardCharsets.UTF_8);
|
||||
assertFalse(writtenLines.isEmpty());
|
||||
assertEquals(StemmerKnowledgeExperiment.ResultRow.csvHeader(), writtenLines.get(0));
|
||||
assertEquals(rows.size() + 1, writtenLines.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the result row key lookup remains stable for all generated
|
||||
* rows.
|
||||
*
|
||||
* @throws IOException if evaluation fails
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("all generated rows should be addressable by the synthetic key")
|
||||
void allGeneratedRowsShouldBeAddressableBySyntheticKey() throws IOException {
|
||||
final StemmerKnowledgeExperiment experiment = new StemmerKnowledgeExperiment();
|
||||
final List<StemmerKnowledgeExperiment.ResultRow> rows = experiment.evaluate(new StringReader(DICTIONARY),
|
||||
"synthetic", "SYNTHETIC", TEST_SEED);
|
||||
|
||||
for (StemmerKnowledgeExperiment.ResultRow row : rows) {
|
||||
assertDoesNotThrow(() -> uniqueRow(rows, resultKey(row)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds one unique row by a synthetic key.
|
||||
*
|
||||
* @param rows result rows
|
||||
* @param key synthetic key
|
||||
* @return matching row
|
||||
*/
|
||||
private static StemmerKnowledgeExperiment.ResultRow uniqueRow(final List<StemmerKnowledgeExperiment.ResultRow> rows,
|
||||
final String key) {
|
||||
final Map<String, StemmerKnowledgeExperiment.ResultRow> indexed = rows.stream()
|
||||
.collect(Collectors.toMap(StemmerKnowledgeExperimentTest::resultKey, Function.identity()));
|
||||
final StemmerKnowledgeExperiment.ResultRow row = indexed.get(key);
|
||||
assertNotNull(row);
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lookup key from a row.
|
||||
*
|
||||
* @param row result row
|
||||
* @return lookup key
|
||||
*/
|
||||
private static String resultKey(final StemmerKnowledgeExperiment.ResultRow row) {
|
||||
return resultKey(ReductionMode.valueOf(row.reductionMode()), row.storeOriginal(), row.includeStemInEvaluation(),
|
||||
row.knowledgePercent());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lookup key from scenario components.
|
||||
*
|
||||
* @param reductionMode reduction mode
|
||||
* @param storeOriginal whether no-op patches were stored
|
||||
* @param includeStemInEvaluation whether stems were evaluated
|
||||
* @param knowledgePercent knowledge percentage
|
||||
* @return lookup key
|
||||
*/
|
||||
private static String resultKey(final ReductionMode reductionMode, final boolean storeOriginal,
|
||||
final boolean includeStemInEvaluation, final int knowledgePercent) {
|
||||
return reductionMode.name() + '|' + storeOriginal + '|' + includeStemInEvaluation + '|' + knowledgePercent;
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -46,13 +47,17 @@ import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.egothor.stemmer.StemmerPatchTrieLoader.Language;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
@@ -77,6 +82,7 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||
* <li>comment-aware parsing delegated to {@link StemmerDictionaryParser}</li>
|
||||
* <li>preservation of all valid stem candidates returned by
|
||||
* {@link FrequencyTrie#getAll(String)}</li>
|
||||
* <li>the current bundled language set, including right-to-left metadata</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Tag("unit")
|
||||
@@ -97,126 +103,51 @@ final class StemmerPatchTrieLoaderTest {
|
||||
*/
|
||||
private static final ReductionMode DEFAULT_REDUCTION_MODE = ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS;
|
||||
|
||||
/**
|
||||
* Representative number of bundled words used for overload consistency checks.
|
||||
*/
|
||||
private static final int REPRESENTATIVE_BUNDLED_WORD_COUNT = 25;
|
||||
|
||||
/**
|
||||
* Provides arguments for bundled dictionary verification across both supported
|
||||
* getAll-preserving reduction modes.
|
||||
*
|
||||
* <p>
|
||||
* The stream is derived directly from the current {@link Language} enum so the
|
||||
* test suite follows the supported bundled language set automatically.
|
||||
* </p>
|
||||
*
|
||||
* @return parameter stream
|
||||
*/
|
||||
static Stream<Arguments> bundledDictionaryCases() {
|
||||
return Stream.of(
|
||||
// 01
|
||||
Arguments.of("01-da_dk-ranked", StemmerPatchTrieLoader.Language.DA_DK,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
final ReductionMode[] reductionModes = new ReductionMode[] {
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS };
|
||||
|
||||
// 02
|
||||
Arguments.of("02-de_de-ranked", StemmerPatchTrieLoader.Language.DE_DE,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 03
|
||||
Arguments.of("03-es_es-ranked", StemmerPatchTrieLoader.Language.ES_ES,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 04
|
||||
Arguments.of("04-fr_fr-ranked", StemmerPatchTrieLoader.Language.FR_FR,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 05
|
||||
Arguments.of("05-it_it-ranked", StemmerPatchTrieLoader.Language.IT_IT,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 06
|
||||
Arguments.of("06-nl_nl-ranked", StemmerPatchTrieLoader.Language.NL_NL,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 07
|
||||
Arguments.of("07-no_no-ranked", StemmerPatchTrieLoader.Language.NO_NO,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 08
|
||||
Arguments.of("08-pt_pt-ranked", StemmerPatchTrieLoader.Language.PT_PT,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 09
|
||||
Arguments.of("09-ru_ru-ranked", StemmerPatchTrieLoader.Language.RU_RU,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 10
|
||||
Arguments.of("10-sv_se-ranked", StemmerPatchTrieLoader.Language.SV_SE,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 11
|
||||
Arguments.of("11-us_uk-ranked", StemmerPatchTrieLoader.Language.US_UK,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 12
|
||||
Arguments.of("12-us_uk_profi-ranked", StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS),
|
||||
|
||||
// 13
|
||||
Arguments.of("13-da_dk-unordered", StemmerPatchTrieLoader.Language.DA_DK,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 14
|
||||
Arguments.of("14-de_de-unordered", StemmerPatchTrieLoader.Language.DE_DE,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 15
|
||||
Arguments.of("15-es_es-unordered", StemmerPatchTrieLoader.Language.ES_ES,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 16
|
||||
Arguments.of("16-fr_fr-unordered", StemmerPatchTrieLoader.Language.FR_FR,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 17
|
||||
Arguments.of("17-it_it-unordered", StemmerPatchTrieLoader.Language.IT_IT,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 18
|
||||
Arguments.of("18-nl_nl-unordered", StemmerPatchTrieLoader.Language.NL_NL,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 19
|
||||
Arguments.of("19-no_no-unordered", StemmerPatchTrieLoader.Language.NO_NO,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 20
|
||||
Arguments.of("20-pt_pt-unordered", StemmerPatchTrieLoader.Language.PT_PT,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 21
|
||||
Arguments.of("21-ru_ru-unordered", StemmerPatchTrieLoader.Language.RU_RU,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 22
|
||||
Arguments.of("22-sv_se-unordered", StemmerPatchTrieLoader.Language.SV_SE,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 23
|
||||
Arguments.of("23-us_uk-unordered", StemmerPatchTrieLoader.Language.US_UK,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS),
|
||||
|
||||
// 24
|
||||
Arguments.of("24-us_uk_profi-unordered", StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_UNORDERED_GET_ALL_RESULTS));
|
||||
return Arrays.stream(StemmerPatchTrieLoader.Language.values())
|
||||
.flatMap(
|
||||
language -> IntStream.range(0, reductionModes.length)
|
||||
.mapToObj(index -> Arguments.of(
|
||||
String.format("%02d-%s-%s", index + 1, language.name().toLowerCase(),
|
||||
reductionModes[index].name().toLowerCase()),
|
||||
language, reductionModes[index])));
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides representative bundled languages for overload consistency checks.
|
||||
*
|
||||
* <p>
|
||||
* The sample intentionally covers both traversal directions.
|
||||
* </p>
|
||||
*
|
||||
* @return parameter stream
|
||||
*/
|
||||
static Stream<Arguments> bundledLanguageSamples() {
|
||||
return Stream.of(
|
||||
// 01
|
||||
Arguments.of("01-us_uk", StemmerPatchTrieLoader.Language.US_UK),
|
||||
|
||||
// 02
|
||||
return Stream.of(Arguments.of("01-us_uk", StemmerPatchTrieLoader.Language.US_UK),
|
||||
Arguments.of("02-de_de", StemmerPatchTrieLoader.Language.DE_DE),
|
||||
|
||||
// 03
|
||||
Arguments.of("03-fr_fr", StemmerPatchTrieLoader.Language.FR_FR));
|
||||
Arguments.of("03-fa_ir", StemmerPatchTrieLoader.Language.FA_IR),
|
||||
Arguments.of("04-he_il", StemmerPatchTrieLoader.Language.HE_IL),
|
||||
Arguments.of("05-yi", StemmerPatchTrieLoader.Language.YI));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -227,107 +158,90 @@ final class StemmerPatchTrieLoaderTest {
|
||||
static Stream<Arguments> nullContractCases() {
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||
final FrequencyTrie<String> trie = new FrequencyTrie.Builder<String>(String[]::new, settings)
|
||||
.put("running", new PatchCommandEncoder().encode("running", "run")).build();
|
||||
.put("running", PatchCommandEncoder.builder().build().encode("running", "run")).build();
|
||||
|
||||
return Stream.of(
|
||||
// 01
|
||||
Arguments.of("01-load-language-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((StemmerPatchTrieLoader.Language) null,
|
||||
true, settings),
|
||||
"language"),
|
||||
|
||||
// 02
|
||||
Arguments.of("02-load-language-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((StemmerPatchTrieLoader.Language) null,
|
||||
true, DEFAULT_REDUCTION_MODE),
|
||||
"language"),
|
||||
|
||||
// 03
|
||||
Arguments.of("03-load-language-null-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||
true, (ReductionSettings) null),
|
||||
"reductionSettings"),
|
||||
|
||||
// 04
|
||||
Arguments.of("04-load-language-null-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||
true, (ReductionMode) null),
|
||||
"reductionMode"),
|
||||
|
||||
// 05
|
||||
Arguments.of("05-load-path-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((Path) null, true, settings), "path"),
|
||||
|
||||
// 06
|
||||
Arguments.of("06-load-path-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((Path) null, true,
|
||||
DEFAULT_REDUCTION_MODE),
|
||||
"path"),
|
||||
|
||||
// 07
|
||||
Arguments.of("07-load-path-null-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true,
|
||||
(ReductionSettings) null),
|
||||
"reductionSettings"),
|
||||
|
||||
// 08
|
||||
Arguments.of("08-load-path-null-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (ReductionMode) null),
|
||||
"reductionMode"),
|
||||
|
||||
// 09
|
||||
Arguments.of("09-load-string-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((String) null, true, settings),
|
||||
"fileName"),
|
||||
|
||||
// 10
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("10-load-string-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load((String) null, true,
|
||||
DEFAULT_REDUCTION_MODE),
|
||||
"fileName"),
|
||||
|
||||
// 11
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("11-load-string-null-settings",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||
(ReductionSettings) null),
|
||||
"reductionSettings"),
|
||||
|
||||
// 12
|
||||
Arguments.of("12-load-string-null-mode",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||
(ReductionMode) null),
|
||||
"reductionMode"),
|
||||
|
||||
// 13
|
||||
Arguments.of("13-load-binary-path",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((Path) null), "path"),
|
||||
|
||||
// 14
|
||||
Arguments.of("14-load-binary-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null), "fileName"),
|
||||
|
||||
// 15
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("15-load-binary-stream",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinary((InputStream) null),
|
||||
"inputStream"),
|
||||
|
||||
// 16
|
||||
Arguments.of("16-save-binary-null-trie-path",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath()), "trie"),
|
||||
|
||||
// 17
|
||||
Arguments.of("17-save-binary-null-path",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (Path) null), "path"),
|
||||
|
||||
// 18
|
||||
Arguments.of("18-save-binary-null-trie-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(null, tempPath().toString()),
|
||||
"trie"),
|
||||
|
||||
// 19
|
||||
Arguments.of("19-save-binary-null-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
|
||||
"fileName"));
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("20-load-language-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||
true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("21-load-path-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("22-load-string-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||
(TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("23-load-binary-metadata-path-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
|
||||
Arguments.of("24-load-binary-metadata-string-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("25-load-binary-metadata-stream-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
|
||||
"inputStream"));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -409,9 +323,9 @@ final class StemmerPatchTrieLoaderTest {
|
||||
@DisplayName("Path and String overloads must load equivalent tries")
|
||||
void shouldLoadEquivalentTrieFromPathAndStringOverloads() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
run running runs runner
|
||||
play playing played plays
|
||||
city cities
|
||||
run running runs runner
|
||||
play playing played plays
|
||||
city cities
|
||||
""");
|
||||
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||
@@ -432,6 +346,31 @@ final class StemmerPatchTrieLoaderTest {
|
||||
"run");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that metadata-driven loading keeps all configuration dimensions in
|
||||
* one explicit object and applies them during compilation.
|
||||
*
|
||||
* @throws IOException if the test file cannot be written or read
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Metadata overload must drive case and diacritic normalization")
|
||||
void shouldLoadUsingExplicitMetadataConfiguration() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
mÁma mamA mámě
|
||||
""");
|
||||
final TrieMetadata metadata = TrieMetadata.forCompilation(WordTraversalDirection.BACKWARD,
|
||||
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE), DiacriticProcessingMode.REMOVE,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, true, metadata);
|
||||
|
||||
assertAll(() -> assertEquals(DiacriticProcessingMode.REMOVE, trie.metadata().diacriticProcessingMode()),
|
||||
() -> assertEquals(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
trie.metadata().caseProcessingMode()),
|
||||
() -> assertNotNull(trie.get("MÁMĚ")),
|
||||
() -> assertNotNull(trie.get("mame")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the loader honors {@code storeOriginal=true} by inserting the
|
||||
* canonical no-op patch for the stem itself.
|
||||
@@ -442,7 +381,7 @@ final class StemmerPatchTrieLoaderTest {
|
||||
@DisplayName("storeOriginal=true must make the stem itself resolvable through the no-op patch")
|
||||
void shouldStoreOriginalStemWhenRequested() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
run running runs
|
||||
run running runs
|
||||
""");
|
||||
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, true,
|
||||
@@ -467,8 +406,8 @@ final class StemmerPatchTrieLoaderTest {
|
||||
@DisplayName("storeOriginal=false must not insert the stem itself unless present as a variant elsewhere")
|
||||
void shouldNotStoreOriginalStemWhenDisabled() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
run running runs
|
||||
play playing played plays
|
||||
run running runs
|
||||
play playing played plays
|
||||
""");
|
||||
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, false,
|
||||
@@ -480,6 +419,29 @@ final class StemmerPatchTrieLoaderTest {
|
||||
"Variants must still reconstruct the proper stem.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the loader honors forward traversal for right-to-left
|
||||
* dictionaries loaded from filesystem overloads.
|
||||
*
|
||||
* @throws IOException if the test file cannot be written or read
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Explicit right-to-left loading must use forward traversal semantics")
|
||||
void shouldUseForwardTraversalForExplicitRightToLeftLoading() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
كتب كتابة كتاب
|
||||
""");
|
||||
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, true, settings,
|
||||
WordTraversalDirection.FORWARD);
|
||||
|
||||
assertEquals(WordTraversalDirection.FORWARD, trie.traversalDirection(),
|
||||
"Right-to-left loading must produce a forward-traversed trie.");
|
||||
assertEquals(Set.of("كتب"), reconstructAllStemCandidates(trie, "كتابة"),
|
||||
"Patch reconstruction must use the trie traversal direction.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that comment syntax documented by the loader is effectively honored
|
||||
* through delegated parsing.
|
||||
@@ -492,10 +454,10 @@ final class StemmerPatchTrieLoaderTest {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
# full-line hash comment
|
||||
// full-line slash comment
|
||||
run running runs // inline slash comment
|
||||
play playing played # inline hash comment
|
||||
run running runs // inline slash comment
|
||||
play playing played # inline hash comment
|
||||
|
||||
city cities
|
||||
city cities
|
||||
""");
|
||||
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, true,
|
||||
@@ -518,9 +480,9 @@ final class StemmerPatchTrieLoaderTest {
|
||||
@DisplayName("Binary save and load overloads must preserve trie semantics")
|
||||
void shouldRoundTripBinaryTrieAcrossAllBinaryOverloads() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
run running runs runner
|
||||
city cities
|
||||
study studies studying
|
||||
run running runs runner
|
||||
city cities
|
||||
study studies studying
|
||||
""");
|
||||
final Path binaryFile = tempDir.resolve("stemmer-trie.bin.gz");
|
||||
|
||||
@@ -539,6 +501,15 @@ final class StemmerPatchTrieLoaderTest {
|
||||
assertTriePatchSemanticsEqual(original, fromString, "run", "running", "runner", "cities", "studying");
|
||||
assertTriePatchSemanticsEqual(original, fromStream, "run", "running", "runner", "cities", "studying");
|
||||
}
|
||||
|
||||
final TrieMetadata metadataFromPath = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile);
|
||||
final TrieMetadata metadataFromString = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile.toString());
|
||||
try (InputStream metadataInputStream = new ByteArrayInputStream(binaryBytes)) {
|
||||
final TrieMetadata metadataFromStream = StemmerPatchTrieLoader.loadBinaryMetadata(metadataInputStream);
|
||||
assertAll(() -> assertEquals(original.metadata(), metadataFromPath),
|
||||
() -> assertEquals(original.metadata(), metadataFromString),
|
||||
() -> assertEquals(original.metadata(), metadataFromStream));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -562,6 +533,53 @@ final class StemmerPatchTrieLoaderTest {
|
||||
@DisplayName("Bundled dictionaries")
|
||||
final class BundledDictionaryTests {
|
||||
|
||||
/**
|
||||
* Verifies that the current language enumeration exactly matches the bundled
|
||||
* language set expected by this project revision.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Language enum must expose the current bundled language set")
|
||||
void shouldExposeCurrentBundledLanguageSet() {
|
||||
final Set<StemmerPatchTrieLoader.Language> expectedLanguages = new LinkedHashSet<StemmerPatchTrieLoader.Language>(
|
||||
Arrays.asList(StemmerPatchTrieLoader.Language.CS_CZ, StemmerPatchTrieLoader.Language.DA_DK,
|
||||
StemmerPatchTrieLoader.Language.DE_DE, StemmerPatchTrieLoader.Language.ES_ES,
|
||||
StemmerPatchTrieLoader.Language.FA_IR, StemmerPatchTrieLoader.Language.FI_FI,
|
||||
StemmerPatchTrieLoader.Language.FR_FR, StemmerPatchTrieLoader.Language.HE_IL,
|
||||
StemmerPatchTrieLoader.Language.HU_HU, StemmerPatchTrieLoader.Language.IT_IT,
|
||||
StemmerPatchTrieLoader.Language.NB_NO, StemmerPatchTrieLoader.Language.NL_NL,
|
||||
StemmerPatchTrieLoader.Language.NN_NO, StemmerPatchTrieLoader.Language.PL_PL,
|
||||
StemmerPatchTrieLoader.Language.PT_PT, StemmerPatchTrieLoader.Language.RU_RU,
|
||||
StemmerPatchTrieLoader.Language.SV_SE, StemmerPatchTrieLoader.Language.UK_UA,
|
||||
StemmerPatchTrieLoader.Language.US_UK, StemmerPatchTrieLoader.Language.YI));
|
||||
|
||||
final Set<StemmerPatchTrieLoader.Language> actualLanguages = new LinkedHashSet<StemmerPatchTrieLoader.Language>(
|
||||
Arrays.asList(StemmerPatchTrieLoader.Language.values()));
|
||||
|
||||
assertEquals(expectedLanguages, actualLanguages,
|
||||
"The bundled language enum must match the project's supported language set exactly.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the right-to-left metadata is correctly assigned for the
|
||||
* currently supported bundled languages.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Language enum must mark right-to-left bundled languages correctly")
|
||||
void shouldExposeCorrectRightToLeftMetadata() {
|
||||
final Set<StemmerPatchTrieLoader.Language> expectedRightToLeftLanguages = Set.of(
|
||||
StemmerPatchTrieLoader.Language.FA_IR, StemmerPatchTrieLoader.Language.HE_IL,
|
||||
StemmerPatchTrieLoader.Language.YI);
|
||||
|
||||
for (StemmerPatchTrieLoader.Language language : StemmerPatchTrieLoader.Language.values()) {
|
||||
if (expectedRightToLeftLanguages.contains(language)) {
|
||||
assertTrue(language.isRightToLeft(), () -> language.name() + " must be marked as right-to-left.");
|
||||
} else {
|
||||
assertFalse(language.isRightToLeft(),
|
||||
() -> language.name() + " must not be marked as right-to-left.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that each bundled dictionary compiles into a trie whose
|
||||
* {@link FrequencyTrie#getAll(String)} results still reconstruct exactly the
|
||||
@@ -586,6 +604,8 @@ final class StemmerPatchTrieLoaderTest {
|
||||
|
||||
assertNotNull(trie, "Compiled trie must be created.");
|
||||
assertFalse(expectedStemsByWord.isEmpty(), "Bundled dictionary must not be empty.");
|
||||
assertEquals(language.isRightToLeft() ? WordTraversalDirection.FORWARD : WordTraversalDirection.BACKWARD,
|
||||
trie.traversalDirection(), "Trie traversal direction must match language metadata.");
|
||||
|
||||
for (Map.Entry<String, Set<String>> entry : expectedStemsByWord.entrySet()) {
|
||||
final String word = entry.getKey();
|
||||
@@ -619,13 +639,12 @@ final class StemmerPatchTrieLoaderTest {
|
||||
final FrequencyTrie<String> viaMode = StemmerPatchTrieLoader.load(language, true, DEFAULT_REDUCTION_MODE);
|
||||
|
||||
final Map<String, Set<String>> expectedStemsByWord = readExpectedStems(language);
|
||||
final int verifiedWords = 25;
|
||||
int counter = 0;
|
||||
|
||||
for (Map.Entry<String, Set<String>> entry : expectedStemsByWord.entrySet()) {
|
||||
assertTriePatchSemanticsEqual(viaSettings, viaMode, entry.getKey());
|
||||
counter++;
|
||||
if (counter >= verifiedWords) {
|
||||
if (counter >= REPRESENTATIVE_BUNDLED_WORD_COUNT) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -704,7 +723,7 @@ final class StemmerPatchTrieLoaderTest {
|
||||
}
|
||||
|
||||
for (String patchCommand : patchCommands) {
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand));
|
||||
stems.add(PatchCommandEncoder.apply(word, patchCommand, trie.traversalDirection()));
|
||||
}
|
||||
|
||||
return stems;
|
||||
@@ -743,7 +762,7 @@ final class StemmerPatchTrieLoaderTest {
|
||||
if (inputStream == null) {
|
||||
throw new IOException("Bundled stemmer resource not found: " + resourcePath);
|
||||
}
|
||||
return inputStream;
|
||||
return new GZIPInputStream(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -82,10 +82,10 @@ class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
|
||||
assertTrue(preferredPatch != null && !preferredPatch.isEmpty(),
|
||||
"preferred patch must exist for an observed word.");
|
||||
assertTrue(allPatches.length >= 1, "at least one patch must exist for an observed word.");
|
||||
assertTrue(acceptableStems.contains(PatchCommandEncoder.apply(observedWord, preferredPatch)),
|
||||
assertTrue(acceptableStems.contains(PatchCommandEncoder.apply(observedWord, preferredPatch, trie.traversalDirection())),
|
||||
"preferred patch reconstructed an unexpected stem.");
|
||||
|
||||
final Set<String> producedStems = applyAll(observedWord, allPatches);
|
||||
final Set<String> producedStems = applyAll(trie, observedWord, allPatches);
|
||||
assertTrue(acceptableStems.containsAll(producedStems),
|
||||
"getAll() must not expose a patch that reconstructs an undeclared stem.");
|
||||
|
||||
@@ -125,10 +125,10 @@ class StemmerPatchTrieProperties extends PropertyBasedTestSupport {
|
||||
* @param patches returned patches
|
||||
* @return decoded stem set
|
||||
*/
|
||||
private static Set<String> applyAll(final String source, final String[] patches) {
|
||||
private static Set<String> applyAll(final FrequencyTrie<String> trie, final String source, final String[] patches) {
|
||||
final LinkedHashSet<String> stems = new LinkedHashSet<>();
|
||||
for (String patch : patches) {
|
||||
stems.add(PatchCommandEncoder.apply(source, patch));
|
||||
stems.add(PatchCommandEncoder.apply(source, patch, trie.traversalDirection()));
|
||||
}
|
||||
return stems;
|
||||
}
|
||||
|
||||
74
src/test/java/org/egothor/stemmer/TrieMetadataTest.java
Normal file
74
src/test/java/org/egothor/stemmer/TrieMetadataTest.java
Normal file
@@ -0,0 +1,74 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@Tag("unit")
|
||||
@DisplayName("TrieMetadata")
|
||||
class TrieMetadataTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("Text block roundtrip preserves all persisted fields")
|
||||
void textBlockRoundtripPreservesAllPersistedFields() {
|
||||
final TrieMetadata metadata = new TrieMetadata(5, WordTraversalDirection.FORWARD,
|
||||
new ReductionSettings(ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_DOMINANT_GET_RESULTS, 80, 4),
|
||||
DiacriticProcessingMode.AS_IS, CaseProcessingMode.AS_IS);
|
||||
|
||||
final String textBlock = metadata.toTextBlock();
|
||||
final TrieMetadata parsed = TrieMetadata.fromTextBlock(5, textBlock);
|
||||
|
||||
assertAll(() -> assertEquals(metadata.traversalDirection(), parsed.traversalDirection()),
|
||||
() -> assertEquals(metadata.reductionSettings(), parsed.reductionSettings()),
|
||||
() -> assertEquals(metadata.diacriticProcessingMode(), parsed.diacriticProcessingMode()),
|
||||
() -> assertEquals(metadata.caseProcessingMode(), parsed.caseProcessingMode()),
|
||||
() -> assertTrue(textBlock.contains("rightToLeft=true")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Text block parser rejects malformed input")
|
||||
void textBlockParserRejectsMalformedInput() {
|
||||
assertAll(
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> TrieMetadata.fromTextBlock(5, "unknown-header\nx=y\n")),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> TrieMetadata.fromTextBlock(5, "radixor.metadata.v1\nmissingDelimiter\n")),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> TrieMetadata.fromTextBlock(5, "radixor.metadata.v1\ntraversalDirection=FORWARD\n")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
package org.egothor.stemmer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@Tag("unit")
|
||||
@DisplayName("WordTraversalDirection")
|
||||
class WordTraversalDirectionTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("startIndex follows direction and validates negatives")
|
||||
void startIndexFollowsDirectionAndValidatesNegatives() {
|
||||
assertAll(() -> assertEquals(0, WordTraversalDirection.FORWARD.startIndex(3)),
|
||||
() -> assertEquals(2, WordTraversalDirection.BACKWARD.startIndex(3)),
|
||||
() -> assertEquals(-1, WordTraversalDirection.FORWARD.startIndex(0)),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> WordTraversalDirection.BACKWARD.startIndex(-1)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("logicalIndex maps offsets in both directions")
|
||||
void logicalIndexMapsOffsetsInBothDirections() {
|
||||
assertAll(() -> assertEquals(0, WordTraversalDirection.FORWARD.logicalIndex(4, 0)),
|
||||
() -> assertEquals(3, WordTraversalDirection.BACKWARD.logicalIndex(4, 0)),
|
||||
() -> assertEquals(1, WordTraversalDirection.FORWARD.logicalIndex(4, 1)),
|
||||
() -> assertEquals(2, WordTraversalDirection.BACKWARD.logicalIndex(4, 1)),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> WordTraversalDirection.FORWARD.logicalIndex(-1, 0)),
|
||||
() -> assertThrows(IllegalArgumentException.class,
|
||||
() -> WordTraversalDirection.BACKWARD.logicalIndex(3, 3)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("traversal character conversion preserves and reverses as expected")
|
||||
void traversalCharacterConversionPreservesAndReversesAsExpected() {
|
||||
assertAll(() -> assertArrayEquals(new char[] { 'a', 'b', 'c' },
|
||||
WordTraversalDirection.FORWARD.toTraversalCharacters("abc")),
|
||||
() -> assertArrayEquals(new char[] { 'c', 'b', 'a' },
|
||||
WordTraversalDirection.BACKWARD.toTraversalCharacters("abc")),
|
||||
() -> assertEquals("abc", WordTraversalDirection.FORWARD.traversalPathToLogicalKey("abc")),
|
||||
() -> assertEquals("cba", WordTraversalDirection.BACKWARD.traversalPathToLogicalKey("abc")),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> WordTraversalDirection.FORWARD.toTraversalCharacters(null)),
|
||||
() -> assertThrows(NullPointerException.class,
|
||||
() -> WordTraversalDirection.BACKWARD.traversalPathToLogicalKey(null)));
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,21 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2026, Leo Galambos
|
||||
* All rights reserved.
|
||||
*
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. All advertising materials mentioning features or use of this software must
|
||||
* display the following acknowledgement:
|
||||
* This product includes software developed by the Egothor project.
|
||||
*
|
||||
* 4. Neither the name of the copyright holder nor the names of its contributors
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# full-line remark
|
||||
// full-line slash remark
|
||||
|
||||
run running runs runner // trailing remark
|
||||
walk walking walks walked
|
||||
city cities
|
||||
café cafés
|
||||
play playing played # trailing remark
|
||||
run running runs runner // trailing remark
|
||||
walk walking walks walked
|
||||
city cities
|
||||
café cafés
|
||||
play playing played # trailing remark
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
62f6419ebab324a69e2e4ef9753687326aa20eed4e851a0f2b63a10f50d2eaae branching-en-ranked-no-storeorig.gz
|
||||
977f1f32db87099f3c3883126f7f9b04a464c47d1e1233c67852feb942a64a4a branching-en-ranked-no-storeorig.gz
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
7b65be9ed9ffab418ed2d1fccc219ea6925e192aa27cdefe5c8383570becd28f mini-en-ranked-storeorig.gz
|
||||
4f74cdfce91adb127461707e7e539438c05e38c9fbfcc0f56275b87597b3f121 mini-en-ranked-storeorig.gz
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
7b65be9ed9ffab418ed2d1fccc219ea6925e192aa27cdefe5c8383570becd28f mini-en-unordered-storeorig.gz
|
||||
2c0cffadbbed91831090baa1bf734449caf6416cea4d041ed79f1c901787538e mini-en-unordered-storeorig.gz
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Focused on subtree branching and repeated suffix families
|
||||
connect connected connecting connects connection
|
||||
collect collected collecting collects collection
|
||||
inspect inspected inspecting inspects inspection
|
||||
direct directed directing directs direction
|
||||
connect connected connecting connects connection
|
||||
collect collected collecting collects collection
|
||||
inspect inspected inspecting inspects inspection
|
||||
direct directed directing directs direction
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Basic English sample with remarks and mixed suffix patterns
|
||||
run running runs runner
|
||||
study studies studying
|
||||
city cities
|
||||
fly flies flying
|
||||
stop stopped stopping stops
|
||||
run running runs runner
|
||||
study studies studying
|
||||
city cities
|
||||
fly flies flying
|
||||
stop stopped stopping stops
|
||||
|
||||
Reference in New Issue
Block a user