Compare commits
8 Commits
release@2.
...
release@2.
| Author | SHA1 | Date | |
|---|---|---|---|
50c3ab3432
|
|||
|
6ccce248ea
|
|||
|
5a511374f3
|
|||
|
48f21cab72
|
|||
|
39969463a2
|
|||
|
6dbdb4bae8
|
|||
|
2ab3e74048
|
|||
|
128fa919f2
|
46
.classpath
46
.classpath
@@ -1,46 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="bin/main" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="main"/>
|
||||
<attribute name="gradle_used_by_scope" value="main,test,jmh"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/test" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="test"/>
|
||||
<attribute name="gradle_used_by_scope" value="test,jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/main" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="main"/>
|
||||
<attribute name="gradle_used_by_scope" value="main,test,jmh"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/jmh" path="src/jmh/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="jmh"/>
|
||||
<attribute name="gradle_used_by_scope" value="jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/jmh" path="build/third-party/snowball/source/libstemmer_java-3.0.1/java">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="jmh"/>
|
||||
<attribute name="gradle_used_by_scope" value="jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="bin/test" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="gradle_scope" value="test"/>
|
||||
<attribute name="gradle_used_by_scope" value="test,jmh"/>
|
||||
<attribute name="test" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-21/"/>
|
||||
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
|
||||
<classpathentry kind="output" path="bin/default"/>
|
||||
</classpath>
|
||||
29
.project
29
.project
@@ -1,23 +1,22 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>Radixor</name>
|
||||
<comment>Project Radixor created by Buildship.</comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<comment></comment>
|
||||
<projects/>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
|
||||
</natures>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
|
||||
<arguments/>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<linkedResources/>
|
||||
<filteredResources/>
|
||||
</projectDescription>
|
||||
|
||||
@@ -54,7 +54,7 @@ Radixor is especially attractive when you want something more adaptable than sim
|
||||
|
||||
Radixor includes a JMH benchmark suite for both its own algorithmic core and a side-by-side English comparison against the Snowball Porter stemmer family.
|
||||
|
||||
On the current English comparison workload, Radixor with bundled `US_UK_PROFI` reaches approximately **31 to 32 million tokens per second**. Snowball original Porter reaches approximately **8 million tokens per second**, and Snowball English (Porter2) approximately **5 to 5.5 million tokens per second**.
|
||||
On the current English comparison workload, Radixor with bundled `US_UK` reaches approximately **31 to 32 million tokens per second**. Snowball original Porter reaches approximately **8 million tokens per second**, and Snowball English (Porter2) approximately **5 to 5.5 million tokens per second**.
|
||||
|
||||
That places Radixor at approximately:
|
||||
|
||||
@@ -137,7 +137,7 @@ The repository keeps the front page concise and places detailed documentation un
|
||||
A practical first guide to loading, compiling, and using Radixor.
|
||||
|
||||
- [Built-in Languages](docs/built-in-languages.md)
|
||||
Overview of bundled language resources such as `US_UK` and `US_UK_PROFI`.
|
||||
Overview of bundled language resources such as `US_UK`.
|
||||
|
||||
- [Dictionary Format](docs/dictionary-format.md)
|
||||
How to write and normalize stemming dictionaries.
|
||||
|
||||
11
build.gradle
11
build.gradle
@@ -33,6 +33,9 @@ configurations {
|
||||
java {
|
||||
withSourcesJar()
|
||||
withJavadocJar()
|
||||
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
}
|
||||
|
||||
tasks.withType(AbstractArchiveTask).configureEach {
|
||||
@@ -51,10 +54,6 @@ pmd {
|
||||
ruleSetFiles = files(rootProject.file(".ruleset"))
|
||||
}
|
||||
|
||||
tasks.withType(JavaCompile).configureEach {
|
||||
options.release = 21
|
||||
}
|
||||
|
||||
dependencyLocking {
|
||||
lockAllConfigurations()
|
||||
|
||||
@@ -237,6 +236,10 @@ distributions {
|
||||
into ''
|
||||
}
|
||||
|
||||
from('LICENSE-stemmer-data') {
|
||||
into ''
|
||||
}
|
||||
|
||||
from('docs') {
|
||||
into 'docs'
|
||||
include '**/*.md'
|
||||
|
||||
@@ -13,7 +13,7 @@ The benchmark suite currently covers two categories:
|
||||
|
||||
The comparison benchmark processes the same deterministic English token stream through:
|
||||
|
||||
- Radixor with bundled `US_UK_PROFI`,
|
||||
- Radixor with bundled `US_UK` (older benchmark snapshots used the now-retired `US_UK_PROFI` resource),
|
||||
- Snowball original Porter,
|
||||
- Snowball English, commonly referred to as Porter2.
|
||||
|
||||
@@ -37,7 +37,7 @@ For that reason, the published badge values should be treated primarily as a com
|
||||
|
||||
A recent JMH run on JDK 21.0.10 with JMH 1.37, one thread, three warmup iterations, and five measurement iterations produced the following approximate throughput ranges:
|
||||
|
||||
| Workload | Radixor `US_UK_PROFI` | Snowball Porter | Snowball English |
|
||||
| Workload | Radixor `US_UK` *(historical runs: `US_UK_PROFI`)* | Snowball Porter | Snowball English |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| About 12,000 generated tokens | 30.99 M tokens/s | 8.21 M tokens/s | 5.46 M tokens/s |
|
||||
| About 60,000 generated tokens | 32.25 M tokens/s | 8.02 M tokens/s | 5.11 M tokens/s |
|
||||
@@ -83,7 +83,7 @@ The workload intentionally mixes:
|
||||
- simple inflections,
|
||||
- common derivational forms,
|
||||
- US and UK spelling families,
|
||||
- lexical forms appropriate for `US_UK_PROFI`.
|
||||
- lexical forms appropriate for the current bundled `US_UK` resource (with historical continuity from earlier `US_UK_PROFI` runs).
|
||||
|
||||
This design keeps runs reproducible across environments and avoids accidental drift caused by changing external corpora.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ This is the preferred preparation workflow when stemming should run against an a
|
||||
|
||||
The `Compile` tool performs the following steps:
|
||||
|
||||
1. reads the input dictionary in the standard Radixor stemmer format,
|
||||
1. reads the input dictionary in the standard Radixor stemmer format, accepting either plain UTF-8 text or GZip-compressed UTF-8 text,
|
||||
2. parses each line into a canonical stem column and its known variant columns,
|
||||
3. converts variants into patch commands,
|
||||
4. builds a mutable trie of patch-command values,
|
||||
@@ -50,7 +50,7 @@ The CLI supports the following arguments:
|
||||
|
||||
Path to the source dictionary file.
|
||||
|
||||
The file must use the standard line-oriented tab-separated values dictionary format, meaning that columns are separated by the tab character. Each non-empty logical line starts with the canonical stem column and may contain zero or more variant columns. The parser expects UTF-8 input, processes case according to `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores trailing remarks introduced by `#` or `//`, and currently ignores dictionary items containing embedded whitespace while reporting them through warning-level log entries.
|
||||
The file must use the standard line-oriented tab-separated values dictionary format, meaning that columns are separated by the tab character. Each non-empty logical line starts with the canonical stem column and may contain zero or more variant columns. The input may be plain UTF-8 text or GZip-compressed UTF-8 text; compression is detected from the stream header rather than the file extension. The parser processes case according to `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores trailing remarks introduced by `#` or `//`, and currently ignores dictionary items containing embedded whitespace while reporting them through warning-level log entries.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -110,7 +110,7 @@ This option is intended for right-to-left languages where affix behavior should
|
||||
|
||||
### `--case-processing-mode <mode>`
|
||||
|
||||
Controls dictionary key normalization during compilation and lookup.
|
||||
Controls dictionary key normalization during compilation and lookup. The setting is stored in persisted trie metadata and is therefore available to runtime lookup after binary loading.
|
||||
|
||||
Supported values are:
|
||||
|
||||
@@ -205,7 +205,7 @@ The CLI is best used as a preparation step during packaging, deployment, or cont
|
||||
|
||||
A `.radixor.gz` file should be handled as a versioned output artifact. It represents a specific dictionary state, a specific reduction mode, and, where relevant, specific dominant-result thresholds.
|
||||
|
||||
Compiled tries also persist a human-readable metadata block (`key=value` lines) that includes traversal direction, RTL indicator, reduction mode, case-processing mode, and dominant thresholds. After decompression, you can inspect this block directly to identify what dictionary/trie configuration the artifact contains.
|
||||
Compiled tries also persist a human-readable metadata block (`key=value` lines) that includes format version, traversal direction, RTL indicator, reduction mode, dominant thresholds, diacritic-processing mode, and case-processing mode. After decompression, you can inspect this block directly to identify what dictionary/trie configuration the artifact contains. The current CLI uses `DiacriticProcessingMode.AS_IS`; custom diacritic stripping is available through the programmatic builder and loader APIs rather than through a CLI flag.
|
||||
|
||||
### Choose reduction mode deliberately
|
||||
|
||||
|
||||
@@ -127,15 +127,21 @@ is processed the same way as:
|
||||
run running runs ran
|
||||
```
|
||||
|
||||
## Character set and practical convention
|
||||
## Character set, compression, and normalization
|
||||
|
||||
Dictionary files are read as UTF-8 text.
|
||||
Dictionary files are read as UTF-8 text. Files loaded through `StemmerPatchTrieLoader.load(Path, ...)` may be either plain UTF-8 text or GZip-compressed UTF-8 text; the loader detects GZip input from the stream header instead of relying on the file extension. Bundled dictionaries are stored as GZip resources and are decoded as UTF-8 after decompression.
|
||||
|
||||
From the perspective of the parser and the stemming algorithm, the format is not restricted to plain ASCII tokens. The parser accepts ordinary Java `String` data, and the trie itself works with general character sequences rather than with an ASCII-only internal model. In principle, this means the system could process diacritic and non-diacritic forms alike, and it could also store forms with inconsistently used diacritics.
|
||||
The parser and trie are not restricted to ASCII. Dictionary items are ordinary Java `String` values, and trie traversal works over Java `char` sequences. This supports Latin-script data with diacritics, Cyrillic data, Hebrew, Persian, Yiddish, and other scripts represented in UTF-8, subject to the normal Java `String` model and the project’s traversal configuration.
|
||||
|
||||
In practice, however, the format is currently best understood as **primarily intended for classical basic ASCII lexical input**, especially in the traditional stemming style where language data is normalized into plain characters in the ASCII range up to character code 127. This convention is particularly relevant for languages whose original orthography includes diacritics but whose stemming dictionaries are commonly maintained in normalized non-diacritic form.
|
||||
Case normalization is controlled by `CaseProcessingMode`. The default `LOWERCASE_WITH_LOCALE_ROOT` mode lowercases the line before columns are split into dictionary items. `AS_IS` preserves the original casing.
|
||||
|
||||
Future versions may expand the documentation and operational guidance for dictionaries that intentionally preserve diacritics. At present, that workflow is not the primary documented use case, not because the algorithm fundamentally forbids it, but because a concrete project requirement for such support has not yet emerged.
|
||||
Diacritic normalization is controlled at trie-build and lookup time by `DiacriticProcessingMode`:
|
||||
|
||||
- `AS_IS` preserves dictionary and lookup keys exactly after case handling,
|
||||
- `REMOVE` strips supported diacritics and common Latin ligatures on both insertion and lookup paths,
|
||||
- `AS_IS_AND_STRIPPED_FALLBACK` is declared in the public model but is not implemented yet and raises `UnsupportedOperationException`.
|
||||
|
||||
For reliable production behavior, choose one normalization policy deliberately and apply it consistently. Normalized ASCII dictionaries remain a practical convention for some legacy stemming data, but they are not a format requirement.
|
||||
|
||||
## Distinct stem and variant semantics
|
||||
|
||||
@@ -206,7 +212,7 @@ The current dictionary format intentionally stays minimal:
|
||||
- no explicit ambiguity syntax,
|
||||
- no sectioning or nested structure.
|
||||
|
||||
Each dictionary item is simply one tab-separated word form after remark stripping and lowercasing.
|
||||
Each dictionary item is simply one tab-separated word form after remark stripping and the configured case and diacritic normalization.
|
||||
|
||||
## Authoring guidance
|
||||
|
||||
@@ -218,7 +224,7 @@ For reliable results, keep dictionaries:
|
||||
- encoded in UTF-8,
|
||||
- easy to audit in plain text form.
|
||||
|
||||
For most current deployments, it is sensible to keep dictionary content in normalized basic ASCII form unless there is a clear requirement to preserve diacritics end-to-end.
|
||||
For most deployments, it is sensible to choose either preserved UTF-8 forms or a normalized ASCII/diacritic-stripped convention and keep that choice consistent across dictionary authoring, compilation, and runtime lookup.
|
||||
|
||||
## Relationship to other documentation
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ public final class BundledLanguageExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ The `storeOriginal` flag controls whether the canonical stem is inserted as a no
|
||||
|
||||
## Load a textual dictionary
|
||||
|
||||
Loading from a dictionary file follows the same preparation model as bundled resources, but the source comes from your own file or path. The textual format is tab-separated values, meaning that columns are separated by the tab character. Each non-empty logical line starts with the stem column and may contain zero or more variant columns. Input case normalization is controlled by `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), trailing remarks introduced by `#` or `//` are ignored, and dictionary items containing embedded whitespace are currently ignored with warning-level diagnostics.
|
||||
Loading from a dictionary file follows the same preparation model as bundled resources, but the source comes from your own file or path. The input may be plain UTF-8 text or GZip-compressed UTF-8 text; the loader detects GZip data from the stream header. The textual format is tab-separated values, meaning that columns are separated by the tab character. Each non-empty logical line starts with the stem column and may contain zero or more variant columns. Input case normalization is controlled by `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), trailing remarks introduced by `#` or `//` are ignored, and dictionary items containing embedded whitespace are currently ignored with warning-level diagnostics.
|
||||
|
||||
```java
|
||||
import java.io.IOException;
|
||||
@@ -59,6 +59,8 @@ public final class LoadTextDictionaryExample {
|
||||
}
|
||||
```
|
||||
|
||||
Additional `StemmerPatchTrieLoader.load(...)` overloads let callers provide explicit `WordTraversalDirection`, `CaseProcessingMode`, `DiacriticProcessingMode`, or a complete `TrieMetadata` instance. Use those overloads when a custom dictionary must be compiled with forward traversal for right-to-left languages, case-sensitive keys, or diacritic stripping.
|
||||
|
||||
## Load a compiled binary artifact
|
||||
|
||||
Binary loading is typically the preferred runtime path because it avoids reparsing the textual source and skips the preparation step entirely.
|
||||
@@ -83,7 +85,7 @@ public final class LoadBinaryExample {
|
||||
}
|
||||
```
|
||||
|
||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression.
|
||||
The binary format is the native `FrequencyTrie` serialization wrapped in GZip compression. It includes persisted `TrieMetadata`, so lookup after loading uses the traversal, case-processing, diacritic-processing, and reduction settings captured when the trie was compiled.
|
||||
|
||||
## Build directly with a mutable builder
|
||||
|
||||
@@ -108,7 +110,7 @@ public final class BuilderExample {
|
||||
final FrequencyTrie.Builder<String> builder =
|
||||
new FrequencyTrie.Builder<>(String[]::new, settings);
|
||||
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
builder.put("running", encoder.encode("running", "run"));
|
||||
builder.put("runs", encoder.encode("runs", "run"));
|
||||
|
||||
@@ -32,7 +32,7 @@ public final class BundledStemmerExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
@@ -69,7 +69,7 @@ public final class LoadBinaryStemmerExample {
|
||||
|
||||
### Build or extend a stemmer from dictionary data
|
||||
|
||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace, and supports line remarks introduced by `#` or `//`.
|
||||
Radixor can also build a compiled trie from a custom dictionary. Dictionary lines consist of a canonical stem followed by zero or more variants. The input may be plain UTF-8 text or GZip-compressed UTF-8 text when loaded from a filesystem path. The parser applies `CaseProcessingMode` (default: `LOWERCASE_WITH_LOCALE_ROOT`), ignores leading and trailing whitespace around columns, supports line remarks introduced by `#` or `//`, and skips dictionary items that contain embedded whitespace.
|
||||
|
||||
This path is also relevant when you extend an existing compiled stemmer with additional domain-specific entries and rebuild a new compact artifact.
|
||||
|
||||
@@ -104,7 +104,7 @@ public final class SingleStemExample {
|
||||
|
||||
public static void main(final String[] arguments) throws IOException {
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(
|
||||
StemmerPatchTrieLoader.Language.US_UK_PROFI,
|
||||
StemmerPatchTrieLoader.Language.US_UK,
|
||||
true,
|
||||
ReductionMode.MERGE_SUBTREES_WITH_EQUIVALENT_RANKED_GET_ALL_RESULTS);
|
||||
|
||||
|
||||
@@ -17,3 +17,6 @@ pomScmDeveloperConnection=scm:git:ssh://git@github.com/leogalambos/Radixor.git
|
||||
|
||||
pomLicenseName=BSD-3-Clause
|
||||
pomLicenseUrl=https://spdx.org/licenses/BSD-3-Clause.html
|
||||
|
||||
pomStemmerDataLicenseName=Stemmer Data License Policy
|
||||
pomStemmerDataLicenseUrl=https://github.com/leogalambos/Radixor/blob/main/LICENSE-stemmer-data
|
||||
|
||||
@@ -13,6 +13,12 @@ def pomScmDeveloperConnection = providers.gradleProperty('pomScmDeveloperConnect
|
||||
def pomLicenseName = providers.gradleProperty('pomLicenseName').orNull
|
||||
def pomLicenseUrl = providers.gradleProperty('pomLicenseUrl').orNull
|
||||
def pomLicenseDistribution = providers.gradleProperty('pomLicenseDistribution').orElse('repo').get()
|
||||
def pomStemmerDataLicenseName = providers.gradleProperty('pomStemmerDataLicenseName')
|
||||
.orElse('Stemmer Data License Policy')
|
||||
.get()
|
||||
def pomStemmerDataLicenseUrl = providers.gradleProperty('pomStemmerDataLicenseUrl')
|
||||
.orElse('https://github.com/leogalambos/Radixor/blob/main/LICENSE-stemmer-data')
|
||||
.get()
|
||||
def pomDeveloperId = providers.gradleProperty('pomDeveloperId').orElse('egothor').get()
|
||||
def pomDeveloperName = providers.gradleProperty('pomDeveloperName').orElse('Leo Galambos').get()
|
||||
def pomDeveloperEmail = providers.gradleProperty('pomDeveloperEmail').orElse('egothor@gmail.com').get()
|
||||
@@ -45,6 +51,11 @@ publishing {
|
||||
url = pomLicenseUrl
|
||||
distribution = pomLicenseDistribution
|
||||
}
|
||||
license {
|
||||
name = pomStemmerDataLicenseName
|
||||
url = pomStemmerDataLicenseUrl
|
||||
distribution = pomLicenseDistribution
|
||||
}
|
||||
}
|
||||
|
||||
developers {
|
||||
@@ -93,6 +104,8 @@ tasks.register('validateReleaseMetadata') {
|
||||
if (pomScmDeveloperConnection == null || pomScmDeveloperConnection.isBlank()) missing.add('pomScmDeveloperConnection')
|
||||
if (pomLicenseName == null || pomLicenseName.isBlank()) missing.add('pomLicenseName')
|
||||
if (pomLicenseUrl == null || pomLicenseUrl.isBlank()) missing.add('pomLicenseUrl')
|
||||
if (pomStemmerDataLicenseName == null || pomStemmerDataLicenseName.isBlank()) missing.add('pomStemmerDataLicenseName')
|
||||
if (pomStemmerDataLicenseUrl == null || pomStemmerDataLicenseUrl.isBlank()) missing.add('pomStemmerDataLicenseUrl')
|
||||
if (signingKey == null || signingKey.isBlank()) missing.add('pomSigningKey / SIGNING_KEY')
|
||||
if (signingPassword == null || signingPassword.isBlank()) missing.add('pomSigningPassword / SIGNING_PASSWORD')
|
||||
|
||||
|
||||
@@ -1,10 +1,27 @@
|
||||
import org.gradle.plugins.ide.eclipse.model.SourceFolder
|
||||
|
||||
|
||||
def snowballVersion = '3.0.1'
|
||||
def snowballArchiveName = "libstemmer_java-${snowballVersion}.tar.gz"
|
||||
def snowballDistributionDirectoryName = "libstemmer_java-${snowballVersion}"
|
||||
def snowballRootRelativePath = 'third-party/snowball'
|
||||
def snowballSourceRelativePath = "${snowballRootRelativePath}/source"
|
||||
def snowballJavaSourceRelativePath = "${snowballSourceRelativePath}/${snowballDistributionDirectoryName}/java"
|
||||
def snowballDownloadUrl = "https://snowballstem.org/dist/${snowballArchiveName}"
|
||||
def snowballDownloadFile = layout.buildDirectory.file("third-party/snowball/${snowballArchiveName}")
|
||||
def snowballExtractDirectory = layout.buildDirectory.dir('third-party/snowball/source')
|
||||
def snowballJavaSourceDirectory = layout.buildDirectory.dir(
|
||||
"third-party/snowball/source/libstemmer_java-${snowballVersion}/java")
|
||||
def snowballDownloadFile = layout.buildDirectory.file("${snowballRootRelativePath}/${snowballArchiveName}")
|
||||
def snowballExtractDirectory = layout.buildDirectory.dir(snowballSourceRelativePath)
|
||||
def snowballJavaSourceDirectory = layout.buildDirectory.dir(snowballJavaSourceRelativePath)
|
||||
def snowballJavaSourceClasspathPath = provider {
|
||||
project.relativePath(snowballJavaSourceDirectory.get().asFile)
|
||||
}
|
||||
def snowballEclipseClasspathAttributes = [
|
||||
gradle_scope : 'jmh',
|
||||
gradle_used_by_scope: 'jmh',
|
||||
test : 'true'
|
||||
]
|
||||
def isAbsoluteClasspathPath = { String path ->
|
||||
path.startsWith('/') || path ==~ /^[A-Za-z]:[\\\/].*/
|
||||
}
|
||||
|
||||
tasks.register('downloadSnowballJava') {
|
||||
group = 'build setup'
|
||||
@@ -47,3 +64,30 @@ sourceSets {
|
||||
tasks.named('compileJmhJava') {
|
||||
dependsOn(tasks.named('extractSnowballJava'))
|
||||
}
|
||||
|
||||
eclipse {
|
||||
classpath {
|
||||
file {
|
||||
whenMerged { classpath ->
|
||||
String generatedSnowballPath = snowballJavaSourceClasspathPath.get()
|
||||
String modelSnowballPath = snowballJavaSourceRelativePath
|
||||
|
||||
classpath.entries.removeAll { entry ->
|
||||
entry.hasProperty('path') && (
|
||||
entry.path == generatedSnowballPath ||
|
||||
entry.path == modelSnowballPath ||
|
||||
isAbsoluteClasspathPath(entry.path)
|
||||
)
|
||||
}
|
||||
|
||||
SourceFolder snowballEntry = new SourceFolder(generatedSnowballPath, null)
|
||||
snowballEntry.output = 'bin/jmh'
|
||||
snowballEclipseClasspathAttributes.each { String name, String value ->
|
||||
snowballEntry.entryAttributes[name] = value
|
||||
}
|
||||
|
||||
classpath.entries.add(snowballEntry)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ final class BenchmarkCorpusSupport {
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, reductionSettings);
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
StemmerDictionaryParser.parse(
|
||||
new StringReader(corpusText),
|
||||
|
||||
@@ -61,6 +61,7 @@ import java.util.logging.Logger;
|
||||
* --output <file>
|
||||
* --reduction-mode <mode>
|
||||
* [--store-original]
|
||||
* [--right-to-left]
|
||||
* [--case-processing-mode <mode>]
|
||||
* [--dominant-winner-min-percent <1..100>]
|
||||
* [--dominant-winner-over-second-ratio <1..n>]
|
||||
|
||||
@@ -85,10 +85,25 @@ final class DiacriticStripper {
|
||||
registerSingle("Þ", 'T');
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility class.
|
||||
*/
|
||||
private DiacriticStripper() {
|
||||
throw new AssertionError("No instances.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes supported diacritic marks and common Latin ligatures from the supplied
|
||||
* text.
|
||||
*
|
||||
* <p>
|
||||
* The method returns the original {@link String} instance when no replacement is
|
||||
* required, avoiding an unnecessary allocation on the common ASCII path.
|
||||
* </p>
|
||||
*
|
||||
* @param input text to normalize
|
||||
* @return normalized text, or {@code input} itself when it is already unchanged
|
||||
*/
|
||||
/* default */ static String strip(final String input) {
|
||||
StringBuilder normalized = null;
|
||||
|
||||
@@ -116,6 +131,13 @@ final class DiacriticStripper {
|
||||
return normalized.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the replacement text for one non-ASCII character.
|
||||
*
|
||||
* @param source source character
|
||||
* @return replacement text, or {@code null} when the character should be kept
|
||||
* unchanged
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
private static String replacementFor(final char source) {
|
||||
if (source <= 0x007F) {
|
||||
@@ -161,6 +183,12 @@ final class DiacriticStripper {
|
||||
return ascii.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers one-character replacements for a set of source characters.
|
||||
*
|
||||
* @param sourceCharacters characters to replace
|
||||
* @param replacement replacement character
|
||||
*/
|
||||
private static void registerSingle(final String sourceCharacters, final char replacement) {
|
||||
for (int index = 0; index < sourceCharacters.length(); index++) {
|
||||
DIRECT_REPLACEMENTS[sourceCharacters.charAt(index)] = replacement;
|
||||
|
||||
@@ -95,21 +95,6 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private static final Logger LOGGER = Logger.getLogger(FrequencyTrie.class.getName());
|
||||
|
||||
/**
|
||||
* Binary format magic header.
|
||||
*/
|
||||
private static final int STREAM_MAGIC = 0x45475452;
|
||||
|
||||
/**
|
||||
* Binary format version.
|
||||
*/
|
||||
private static final int STREAM_VERSION = 5;
|
||||
|
||||
/**
|
||||
* Factory used to create correctly typed arrays for {@link #getAll(String)}.
|
||||
*/
|
||||
private final IntFunction<V[]> arrayFactory;
|
||||
|
||||
/**
|
||||
* Root node of the compiled read-only trie.
|
||||
*/
|
||||
@@ -120,19 +105,67 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private final TrieMetadata metadata;
|
||||
|
||||
/**
|
||||
* Cached traversal direction used for key lookup.
|
||||
*/
|
||||
private final WordTraversalDirection lookupTraversalDirection;
|
||||
|
||||
/**
|
||||
* Whether lookups require lowercase normalization.
|
||||
*/
|
||||
private final boolean lowercasesLookupKeys;
|
||||
|
||||
/**
|
||||
* Whether lookups require diacritic stripping.
|
||||
*/
|
||||
private final boolean removeDiacritics;
|
||||
|
||||
/**
|
||||
* Shared empty array instance for empty lookup results from {@link #getAll(String)}.
|
||||
*/
|
||||
private final V[] emptyValues;
|
||||
|
||||
/**
|
||||
* Binary format magic header.
|
||||
*/
|
||||
private static final int STREAM_MAGIC = 0x45475452;
|
||||
|
||||
/**
|
||||
* Binary format version.
|
||||
*/
|
||||
private static final int STREAM_VERSION = 5;
|
||||
|
||||
/**
|
||||
* Returns the current persisted binary stream format version.
|
||||
*
|
||||
* <p>
|
||||
* This method exists so other components can construct {@link TrieMetadata}
|
||||
* instances aligned with the currently written binary format without
|
||||
* duplicating constants.
|
||||
* </p>
|
||||
*
|
||||
* @return current trie stream format version
|
||||
*/
|
||||
public static int currentFormatVersion() {
|
||||
return STREAM_VERSION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new compiled trie instance.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param root compiled root node
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param metadata trie metadata describing lookup and persistence semantics
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
*/
|
||||
private FrequencyTrie(final IntFunction<V[]> arrayFactory, final CompiledNode<V> root,
|
||||
final TrieMetadata metadata) {
|
||||
this.arrayFactory = Objects.requireNonNull(arrayFactory, "arrayFactory");
|
||||
this.root = Objects.requireNonNull(root, "root");
|
||||
this.metadata = Objects.requireNonNull(metadata, "metadata");
|
||||
this.lookupTraversalDirection = metadata.traversalDirection();
|
||||
this.lowercasesLookupKeys = metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT;
|
||||
this.removeDiacritics = metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE;
|
||||
this.emptyValues = arrayFactory.apply(0);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -157,10 +190,14 @@ public final class FrequencyTrie<V> {
|
||||
public V get(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
if (node == null) {
|
||||
return null;
|
||||
}
|
||||
return node.orderedValues()[0];
|
||||
final V[] orderedValues = node.orderedValues();
|
||||
if (orderedValues.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return orderedValues[0];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -186,13 +223,18 @@ public final class FrequencyTrie<V> {
|
||||
* value is stored at the addressed node
|
||||
* @throws NullPointerException if {@code key} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings("PMD.MethodReturnsInternalArray")
|
||||
public V[] getAll(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
return this.arrayFactory.apply(0);
|
||||
if (node == null) {
|
||||
return this.emptyValues;
|
||||
}
|
||||
return Arrays.copyOf(node.orderedValues(), node.orderedValues().length);
|
||||
final V[] orderedValues = node.orderedValues();
|
||||
if (orderedValues.length == 0) {
|
||||
return this.emptyValues;
|
||||
}
|
||||
return Arrays.copyOf(orderedValues, orderedValues.length);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -217,16 +259,28 @@ public final class FrequencyTrie<V> {
|
||||
* if the key does not exist or no value is stored at the addressed node
|
||||
* @throws NullPointerException if {@code key} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
|
||||
public List<ValueCount<V>> getEntries(final String key) {
|
||||
Objects.requireNonNull(key, "key");
|
||||
final CompiledNode<V> node = findNode(normalizeLookupKey(key));
|
||||
if (node == null || node.orderedValues().length == 0) {
|
||||
if (node == null) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
final List<ValueCount<V>> entries = new ArrayList<>(node.orderedValues().length);
|
||||
for (int index = 0; index < node.orderedValues().length; index++) {
|
||||
entries.add(new ValueCount<>(node.orderedValues()[index], node.orderedCounts()[index]));
|
||||
final V[] orderedValues = node.orderedValues();
|
||||
final int valueCount = orderedValues.length;
|
||||
if (valueCount == 0) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
if (valueCount == 1) {
|
||||
return List.of(new ValueCount<>(orderedValues[0], node.orderedCounts()[0]));
|
||||
}
|
||||
|
||||
final int[] orderedCounts = node.orderedCounts();
|
||||
final List<ValueCount<V>> entries = new ArrayList<>(valueCount);
|
||||
for (int index = 0; index < valueCount; index++) {
|
||||
entries.add(new ValueCount<>(orderedValues[index], orderedCounts[index]));
|
||||
}
|
||||
return Collections.unmodifiableList(entries);
|
||||
}
|
||||
@@ -629,9 +683,18 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
private CompiledNode<V> findNode(final String key) {
|
||||
CompiledNode<V> current = this.root;
|
||||
if (this.lookupTraversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
for (int traversalOffset = key.length() - 1; traversalOffset >= 0; traversalOffset--) {
|
||||
current = current.findChild(key.charAt(traversalOffset));
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
for (int traversalOffset = 0; traversalOffset < key.length(); traversalOffset++) {
|
||||
current = current.findChild(
|
||||
key.charAt(this.metadata.traversalDirection().logicalIndex(key.length(), traversalOffset)));
|
||||
current = current.findChild(key.charAt(traversalOffset));
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
@@ -646,13 +709,15 @@ public final class FrequencyTrie<V> {
|
||||
* @return normalized key for trie traversal
|
||||
*/
|
||||
private String normalizeLookupKey(final String key) {
|
||||
String normalized = key;
|
||||
|
||||
if (this.metadata.caseProcessingMode() == CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT) {
|
||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||
if (!this.lowercasesLookupKeys && !this.removeDiacritics) {
|
||||
return key;
|
||||
}
|
||||
|
||||
if (this.metadata.diacriticProcessingMode() == DiacriticProcessingMode.REMOVE) {
|
||||
String normalized = key;
|
||||
if (this.lowercasesLookupKeys) {
|
||||
normalized = normalized.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
if (this.removeDiacritics) {
|
||||
normalized = DiacriticStripper.strip(normalized);
|
||||
} else if (this.metadata.diacriticProcessingMode() == DiacriticProcessingMode.AS_IS_AND_STRIPPED_FALLBACK) {
|
||||
throw new UnsupportedOperationException(
|
||||
@@ -753,13 +818,14 @@ public final class FrequencyTrie<V> {
|
||||
*/
|
||||
public Builder(final IntFunction<V[]> arrayFactory, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode) {
|
||||
this(arrayFactory, reductionSettings, traversalDirection, caseProcessingMode, DiacriticProcessingMode.AS_IS);
|
||||
this(arrayFactory, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new builder with the provided settings, explicit traversal
|
||||
* direction, explicit case processing mode, and explicit diacritic
|
||||
* processing mode.
|
||||
* direction, explicit case processing mode, and explicit diacritic processing
|
||||
* mode.
|
||||
*
|
||||
* @param arrayFactory array factory
|
||||
* @param reductionSettings reduction configuration
|
||||
@@ -847,8 +913,8 @@ public final class FrequencyTrie<V> {
|
||||
reductionContext.canonicalNodeCount());
|
||||
}
|
||||
|
||||
final TrieMetadata metadata = new TrieMetadata(STREAM_VERSION, this.traversalDirection,
|
||||
this.reductionSettings, this.diacriticProcessingMode, this.caseProcessingMode);
|
||||
final TrieMetadata metadata = TrieMetadata.forCompilation(this.traversalDirection, this.reductionSettings,
|
||||
this.diacriticProcessingMode, this.caseProcessingMode);
|
||||
return new FrequencyTrie<>(this.arrayFactory, compiledRoot, metadata);
|
||||
}
|
||||
|
||||
@@ -906,6 +972,13 @@ public final class FrequencyTrie<V> {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies build-time dictionary-key normalization according to the builder
|
||||
* configuration.
|
||||
*
|
||||
* @param key dictionary key
|
||||
* @return normalized key for trie insertion
|
||||
*/
|
||||
private String normalizeDictionaryKey(final String key) {
|
||||
String normalized = key;
|
||||
|
||||
|
||||
@@ -70,6 +70,16 @@ import java.util.concurrent.locks.ReentrantLock;
|
||||
@SuppressWarnings("PMD.CyclomaticComplexity")
|
||||
public final class PatchCommandEncoder {
|
||||
|
||||
/**
|
||||
* Backward direction apply strategy with no runtime direction branching.
|
||||
*/
|
||||
private static final ApplyStrategy BACKWARD_APPLY_STRATEGY = PatchCommandEncoder::applyBackward;
|
||||
|
||||
/**
|
||||
* Forward direction apply strategy with no runtime direction branching.
|
||||
*/
|
||||
private static final ApplyStrategy FORWARD_APPLY_STRATEGY = PatchCommandEncoder::applyForward;
|
||||
|
||||
/**
|
||||
* Serialized opcode for deleting one or more characters.
|
||||
*/
|
||||
@@ -111,6 +121,16 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
/* default */ static final String NOOP_PATCH = String.valueOf(new char[] { NOOP_OPCODE, NOOP_ARGUMENT });
|
||||
|
||||
/**
|
||||
* Prefix used in unsupported NOOP patch argument exceptions.
|
||||
*/
|
||||
private static final String MSG_NOOP = "Unsupported NOOP patch argument: ";
|
||||
|
||||
/**
|
||||
* Prefix used in unsupported patch opcode exceptions.
|
||||
*/
|
||||
private static final String MSG_OPCODE = "Unsupported patch opcode: ";
|
||||
|
||||
/**
|
||||
* Safety penalty used to prevent a mismatch from being selected as a match.
|
||||
*/
|
||||
@@ -147,6 +167,11 @@ public final class PatchCommandEncoder {
|
||||
*/
|
||||
private final WordTraversalDirection traversalDirection;
|
||||
|
||||
/**
|
||||
* Direction-specialized patch apply strategy.
|
||||
*/
|
||||
private final ApplyStrategy applyStrategy;
|
||||
|
||||
/**
|
||||
* Currently allocated source dimension of reusable matrices.
|
||||
*/
|
||||
@@ -191,56 +216,35 @@ public final class PatchCommandEncoder {
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with the traditional Egothor cost model: insert = 1,
|
||||
* delete = 1, replace = 1, match = 0.
|
||||
* Direction-specialized patch application strategy.
|
||||
*/
|
||||
public PatchCommandEncoder() {
|
||||
this(WordTraversalDirection.BACKWARD, 1, 1, 1, 0);
|
||||
@FunctionalInterface
|
||||
private interface ApplyStrategy {
|
||||
/**
|
||||
* Applies the command.
|
||||
*
|
||||
* @param source original text
|
||||
* @param patchCommand patch command
|
||||
* @return final text after applying the command
|
||||
*/
|
||||
String apply(String source, String patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with the traditional Egothor cost model and explicit
|
||||
* traversal direction.
|
||||
*
|
||||
* @param traversalDirection traversal direction
|
||||
*/
|
||||
public PatchCommandEncoder(final WordTraversalDirection traversalDirection) {
|
||||
this(traversalDirection, 1, 1, 1, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with explicit operation costs.
|
||||
*
|
||||
* @param insertCost cost of inserting one character
|
||||
* @param deleteCost cost of deleting one character
|
||||
* @param replaceCost cost of replacing one character
|
||||
* @param matchCost cost of keeping one equal character unchanged
|
||||
*/
|
||||
public PatchCommandEncoder(final int insertCost, final int deleteCost, final int replaceCost, final int matchCost) {
|
||||
this(WordTraversalDirection.BACKWARD, insertCost, deleteCost, replaceCost, matchCost);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an encoder with explicit operation costs and traversal direction.
|
||||
*
|
||||
* @param traversalDirection traversal direction
|
||||
* @param insertCost cost of inserting one character
|
||||
* @param deleteCost cost of deleting one character
|
||||
* @param replaceCost cost of replacing one character
|
||||
* @param matchCost cost of keeping one equal character unchanged
|
||||
*/
|
||||
public PatchCommandEncoder(final WordTraversalDirection traversalDirection, final int insertCost,
|
||||
final int deleteCost, final int replaceCost, final int matchCost) {
|
||||
this.traversalDirection = Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
private PatchCommandEncoder(final Builder builder) {
|
||||
this.traversalDirection = Objects.requireNonNull(builder.traversalDirection, "traversalDirection");
|
||||
final int insertCost = builder.insertCost;
|
||||
if (insertCost < 0) {
|
||||
throw new IllegalArgumentException("insertCost must be non-negative.");
|
||||
}
|
||||
final int deleteCost = builder.deleteCost;
|
||||
if (deleteCost < 0) {
|
||||
throw new IllegalArgumentException("deleteCost must be non-negative.");
|
||||
}
|
||||
final int replaceCost = builder.replaceCost;
|
||||
if (replaceCost < 0) {
|
||||
throw new IllegalArgumentException("replaceCost must be non-negative.");
|
||||
}
|
||||
final int matchCost = builder.matchCost;
|
||||
if (matchCost < 0) {
|
||||
throw new IllegalArgumentException("matchCost must be non-negative.");
|
||||
}
|
||||
@@ -249,12 +253,22 @@ public final class PatchCommandEncoder {
|
||||
this.deleteCost = deleteCost;
|
||||
this.replaceCost = replaceCost;
|
||||
this.matchCost = matchCost;
|
||||
this.applyStrategy = applyStrategyFor(this.traversalDirection);
|
||||
this.sourceCapacity = 0;
|
||||
this.targetCapacity = 0;
|
||||
this.costMatrix = new int[0][0];
|
||||
this.traceMatrix = new Trace[0][0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a fluent builder for constructing a direction-specialized encoder.
|
||||
*
|
||||
* @return new builder instance
|
||||
*/
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces a compact patch command that transforms {@code source} into
|
||||
* {@code target}.
|
||||
@@ -272,9 +286,30 @@ public final class PatchCommandEncoder {
|
||||
return NOOP_PATCH;
|
||||
}
|
||||
|
||||
final String effectiveSource = toLegacyWordForm(source, this.traversalDirection);
|
||||
final String effectiveTarget = toLegacyWordForm(target, this.traversalDirection);
|
||||
return encodeBackward(effectiveSource, effectiveTarget);
|
||||
if (this.traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
return encodeBackward(source, target);
|
||||
}
|
||||
return encodeForward(source, target);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a compact patch command using this encoder instance traversal
|
||||
* direction.
|
||||
*
|
||||
* <p>
|
||||
* This is the branch-free instance-level fast path for repeated patch
|
||||
* application in a known traversal direction.
|
||||
* </p>
|
||||
*
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
public String applyWithConfiguredDirection(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
return this.applyStrategy.apply(source, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -294,9 +329,7 @@ public final class PatchCommandEncoder {
|
||||
* specified traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* Forward traversal is implemented by transforming the source word to the
|
||||
* equivalent legacy backward form, applying the proven historical decoder, and
|
||||
* reversing the transformed result back to the logical word form.
|
||||
* The implementation uses dedicated direction-specific patch decoders.
|
||||
* </p>
|
||||
*
|
||||
* @param source original source word
|
||||
@@ -310,12 +343,7 @@ public final class PatchCommandEncoder {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
if (traversalDirection == WordTraversalDirection.BACKWARD) {
|
||||
return applyBackward(source, patchCommand);
|
||||
}
|
||||
final String transformedSource = reverse(source);
|
||||
final String transformedResult = applyBackward(transformedSource, patchCommand);
|
||||
return reverse(transformedResult);
|
||||
return applyStrategyFor(traversalDirection).apply(source, patchCommand);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -332,14 +360,43 @@ public final class PatchCommandEncoder {
|
||||
lock.lock();
|
||||
try {
|
||||
ensureCapacity(sourceLength + 1, targetLength + 1);
|
||||
initializeBoundaryConditions(sourceLength, targetLength);
|
||||
initializeBoundaryConditionsBackward(sourceLength, targetLength);
|
||||
|
||||
final char[] sourceCharacters = source.toCharArray();
|
||||
final char[] targetCharacters = target.toCharArray();
|
||||
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength);
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength,
|
||||
WordTraversalDirection.BACKWARD);
|
||||
|
||||
return buildPatchCommand(targetCharacters, sourceLength, targetLength);
|
||||
return buildPatchCommandBackward(targetCharacters, sourceLength, targetLength);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a patch command using forward traversal semantics.
|
||||
*
|
||||
* @param source source word form
|
||||
* @param target target word form
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String encodeForward(final String source, final String target) {
|
||||
final int sourceLength = source.length();
|
||||
final int targetLength = target.length();
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
ensureCapacity(sourceLength + 1, targetLength + 1);
|
||||
initializeBoundaryConditionsForward(sourceLength, targetLength);
|
||||
|
||||
final char[] sourceCharacters = source.toCharArray();
|
||||
final char[] targetCharacters = target.toCharArray();
|
||||
|
||||
fillMatrices(sourceCharacters, targetCharacters, sourceLength, targetLength,
|
||||
WordTraversalDirection.FORWARD);
|
||||
|
||||
return buildPatchCommandForward(targetCharacters, sourceLength, targetLength);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
@@ -366,6 +423,9 @@ public final class PatchCommandEncoder {
|
||||
if ((patchCommand.length() & 1) != 0) {
|
||||
return source;
|
||||
}
|
||||
if (patchCommand.length() == 2) {
|
||||
return applySingleBackwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||
}
|
||||
|
||||
final StringBuilder result = new StringBuilder(source);
|
||||
if (result.isEmpty()) {
|
||||
@@ -426,6 +486,184 @@ public final class PatchCommandEncoder {
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a patch command using forward traversal semantics.
|
||||
*
|
||||
* @param source original source word
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or {@code null} when {@code source} is {@code null}
|
||||
*/
|
||||
@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition" })
|
||||
private static String applyForward(final String source, final String patchCommand) {
|
||||
if (source == null) {
|
||||
return null;
|
||||
}
|
||||
if (patchCommand == null || patchCommand.isEmpty()) {
|
||||
return source;
|
||||
}
|
||||
if (NOOP_PATCH.equals(patchCommand)) {
|
||||
return source;
|
||||
}
|
||||
if ((patchCommand.length() & 1) != 0) {
|
||||
return source;
|
||||
}
|
||||
if (patchCommand.length() == 2) {
|
||||
return applySingleForwardInstruction(source, patchCommand.charAt(0), patchCommand.charAt(1));
|
||||
}
|
||||
|
||||
final StringBuilder result = new StringBuilder(source);
|
||||
if (result.isEmpty()) {
|
||||
return applyForwardToEmptySource(result, patchCommand);
|
||||
}
|
||||
|
||||
int position = 0;
|
||||
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case SKIP_OPCODE:
|
||||
final int skipCount = decodeEncodedCount(argument);
|
||||
if (skipCount < 1) {
|
||||
return source;
|
||||
}
|
||||
position = position + skipCount - 1;
|
||||
break;
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
result.setCharAt(position, argument);
|
||||
break;
|
||||
|
||||
case DELETE_OPCODE:
|
||||
final int deleteCount = decodeEncodedCount(argument);
|
||||
if (deleteCount < 1) {
|
||||
return source;
|
||||
}
|
||||
result.delete(position, position + deleteCount);
|
||||
position--;
|
||||
break;
|
||||
|
||||
case INSERT_OPCODE:
|
||||
result.insert(position, argument);
|
||||
break;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
|
||||
}
|
||||
return source;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
|
||||
}
|
||||
|
||||
position++;
|
||||
}
|
||||
} catch (IndexOutOfBoundsException exception) {
|
||||
return source;
|
||||
}
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a single backward-direction patch instruction.
|
||||
*
|
||||
* @param source original source word
|
||||
* @param opcode patch opcode
|
||||
* @param argument encoded patch argument
|
||||
* @return transformed source after one instruction
|
||||
*/
|
||||
private static String applySingleBackwardInstruction(final String source, final char opcode, final char argument) {
|
||||
final int sourceLength = source.length();
|
||||
final int encodedValue;
|
||||
|
||||
switch (opcode) {
|
||||
case DELETE_OPCODE:
|
||||
encodedValue = decodeEncodedCount(argument);
|
||||
if (encodedValue < 1 || encodedValue > sourceLength) {
|
||||
return source;
|
||||
}
|
||||
return source.substring(0, sourceLength - encodedValue);
|
||||
|
||||
case INSERT_OPCODE:
|
||||
final char[] insertTarget = new char[sourceLength + 1];
|
||||
source.getChars(0, sourceLength, insertTarget, 0);
|
||||
insertTarget[sourceLength] = argument;
|
||||
return new String(insertTarget);
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
if (sourceLength == 0) {
|
||||
return source;
|
||||
}
|
||||
final char[] replaceTarget = source.toCharArray();
|
||||
replaceTarget[sourceLength - 1] = argument;
|
||||
return new String(replaceTarget);
|
||||
|
||||
case SKIP_OPCODE:
|
||||
return source;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return source;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a single forward-direction patch instruction.
|
||||
*
|
||||
* @param source original source word
|
||||
* @param opcode patch opcode
|
||||
* @param argument encoded patch argument
|
||||
* @return transformed source after one instruction
|
||||
*/
|
||||
private static String applySingleForwardInstruction(final String source, final char opcode, final char argument) {
|
||||
final int sourceLength = source.length();
|
||||
final int encodedValue;
|
||||
|
||||
switch (opcode) {
|
||||
case DELETE_OPCODE:
|
||||
encodedValue = decodeEncodedCount(argument);
|
||||
if (encodedValue < 1 || encodedValue > sourceLength) {
|
||||
return source;
|
||||
}
|
||||
return source.substring(encodedValue);
|
||||
|
||||
case INSERT_OPCODE:
|
||||
final char[] insertTarget = new char[sourceLength + 1];
|
||||
insertTarget[0] = argument;
|
||||
source.getChars(0, sourceLength, insertTarget, 1);
|
||||
return new String(insertTarget);
|
||||
|
||||
case REPLACE_OPCODE:
|
||||
if (sourceLength == 0) {
|
||||
return source;
|
||||
}
|
||||
final char[] replaceTarget = source.toCharArray();
|
||||
replaceTarget[0] = argument;
|
||||
return new String(replaceTarget);
|
||||
|
||||
case SKIP_OPCODE:
|
||||
return source;
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException(MSG_NOOP + argument);
|
||||
}
|
||||
return source;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException(MSG_OPCODE + opcode);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a backward patch command to an empty source word.
|
||||
*
|
||||
@@ -475,25 +713,54 @@ public final class PatchCommandEncoder {
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a logical word to the equivalent word form expected by the legacy
|
||||
* backward encoder.
|
||||
* Applies a forward patch command to an empty source word.
|
||||
*
|
||||
* @param word logical word form
|
||||
* @param traversalDirection requested traversal direction
|
||||
* @return word form suitable for the legacy backward algorithm
|
||||
* @param result empty result builder
|
||||
* @param patchCommand compact patch command
|
||||
* @return transformed word, or the original empty word when the patch is
|
||||
* malformed
|
||||
*/
|
||||
private static String toLegacyWordForm(final String word, final WordTraversalDirection traversalDirection) {
|
||||
return traversalDirection == WordTraversalDirection.BACKWARD ? word : reverse(word);
|
||||
private static String applyForwardToEmptySource(final StringBuilder result, final String patchCommand) {
|
||||
try {
|
||||
for (int patchIndex = 0, patchLength = patchCommand.length(); patchIndex < patchLength; patchIndex += 2) { // NOPMD
|
||||
final char opcode = patchCommand.charAt(patchIndex);
|
||||
final char argument = patchCommand.charAt(patchIndex + 1);
|
||||
|
||||
switch (opcode) {
|
||||
case INSERT_OPCODE:
|
||||
result.append(argument);
|
||||
break;
|
||||
|
||||
case SKIP_OPCODE:
|
||||
case REPLACE_OPCODE:
|
||||
case DELETE_OPCODE:
|
||||
return "";
|
||||
|
||||
case NOOP_OPCODE:
|
||||
if (argument != NOOP_ARGUMENT) {
|
||||
throw new IllegalArgumentException("Unsupported NOOP patch argument: " + argument);
|
||||
}
|
||||
return "";
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unsupported patch opcode: " + opcode);
|
||||
}
|
||||
}
|
||||
} catch (IndexOutOfBoundsException exception) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverses the supplied word.
|
||||
* Returns the direction-specialized apply strategy.
|
||||
*
|
||||
* @param word source word
|
||||
* @return reversed word
|
||||
* @param traversalDirection requested traversal direction
|
||||
* @return branch-free apply strategy for that direction
|
||||
*/
|
||||
private static String reverse(final String word) {
|
||||
return new StringBuilder(word).reverse().toString();
|
||||
private static ApplyStrategy applyStrategyFor(final WordTraversalDirection traversalDirection) {
|
||||
return traversalDirection == WordTraversalDirection.BACKWARD ? BACKWARD_APPLY_STRATEGY : FORWARD_APPLY_STRATEGY;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -536,7 +803,7 @@ public final class PatchCommandEncoder {
|
||||
* @param sourceLength length of the source word
|
||||
* @param targetLength length of the target word
|
||||
*/
|
||||
private void initializeBoundaryConditions(final int sourceLength, final int targetLength) {
|
||||
private void initializeBoundaryConditionsBackward(final int sourceLength, final int targetLength) {
|
||||
this.costMatrix[0][0] = 0;
|
||||
this.traceMatrix[0][0] = Trace.MATCH;
|
||||
|
||||
@@ -551,6 +818,29 @@ public final class PatchCommandEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes boundary conditions for forward dynamic-programming traversal.
|
||||
*
|
||||
* @param sourceLength length of the source word
|
||||
* @param targetLength length of the target word
|
||||
*/
|
||||
private void initializeBoundaryConditionsForward(final int sourceLength, final int targetLength) {
|
||||
this.costMatrix[sourceLength][targetLength] = 0;
|
||||
this.traceMatrix[sourceLength][targetLength] = Trace.MATCH;
|
||||
|
||||
for (int sourceIndex = sourceLength - 1; sourceIndex >= 0; sourceIndex--) {
|
||||
this.costMatrix[sourceIndex][targetLength] = this.costMatrix[sourceIndex + 1][targetLength]
|
||||
+ this.deleteCost;
|
||||
this.traceMatrix[sourceIndex][targetLength] = Trace.DELETE;
|
||||
}
|
||||
|
||||
for (int targetIndex = targetLength - 1; targetIndex >= 0; targetIndex--) {
|
||||
this.costMatrix[sourceLength][targetIndex] = this.costMatrix[sourceLength][targetIndex + 1]
|
||||
+ this.insertCost;
|
||||
this.traceMatrix[sourceLength][targetIndex] = Trace.INSERT;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills dynamic-programming matrices for the supplied source and target
|
||||
* character sequences.
|
||||
@@ -559,20 +849,57 @@ public final class PatchCommandEncoder {
|
||||
* @param targetCharacters target characters
|
||||
* @param sourceLength source length
|
||||
* @param targetLength target length
|
||||
* @param direction traversal direction used to compare characters
|
||||
*/
|
||||
private void fillMatrices(final char[] sourceCharacters, final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength) {
|
||||
final int targetLength, final WordTraversalDirection direction) {
|
||||
final int sourceStart;
|
||||
final int sourceEndExclusive;
|
||||
final int sourceStep;
|
||||
final int targetStart;
|
||||
final int targetEndExclusive;
|
||||
final int targetStep;
|
||||
final int sourceCharacterOffset;
|
||||
final int targetCharacterOffset;
|
||||
final int sourceNeighborDelta;
|
||||
final int targetNeighborDelta;
|
||||
|
||||
for (int sourceIndex = 1; sourceIndex <= sourceLength; sourceIndex++) {
|
||||
final char sourceCharacter = sourceCharacters[sourceIndex - 1];
|
||||
if (direction == WordTraversalDirection.BACKWARD) {
|
||||
sourceStart = 1;
|
||||
sourceEndExclusive = sourceLength + 1;
|
||||
sourceStep = 1;
|
||||
targetStart = 1;
|
||||
targetEndExclusive = targetLength + 1;
|
||||
targetStep = 1;
|
||||
sourceCharacterOffset = -1;
|
||||
targetCharacterOffset = -1;
|
||||
sourceNeighborDelta = -1;
|
||||
targetNeighborDelta = -1;
|
||||
} else {
|
||||
sourceStart = sourceLength - 1;
|
||||
sourceEndExclusive = -1;
|
||||
sourceStep = -1;
|
||||
targetStart = targetLength - 1;
|
||||
targetEndExclusive = -1;
|
||||
targetStep = -1;
|
||||
sourceCharacterOffset = 0;
|
||||
targetCharacterOffset = 0;
|
||||
sourceNeighborDelta = 1;
|
||||
targetNeighborDelta = 1;
|
||||
}
|
||||
|
||||
for (int targetIndex = 1; targetIndex <= targetLength; targetIndex++) {
|
||||
final char targetCharacter = targetCharacters[targetIndex - 1];
|
||||
for (int sourceIndex = sourceStart; sourceIndex != sourceEndExclusive; sourceIndex += sourceStep) {
|
||||
final char sourceCharacter = sourceCharacters[sourceIndex + sourceCharacterOffset];
|
||||
final int sourceNeighbor = sourceIndex + sourceNeighborDelta;
|
||||
|
||||
final int deleteCandidate = this.costMatrix[sourceIndex - 1][targetIndex] + this.deleteCost;
|
||||
final int insertCandidate = this.costMatrix[sourceIndex][targetIndex - 1] + this.insertCost;
|
||||
final int replaceCandidate = this.costMatrix[sourceIndex - 1][targetIndex - 1] + this.replaceCost;
|
||||
final int matchCandidate = this.costMatrix[sourceIndex - 1][targetIndex - 1]
|
||||
for (int targetIndex = targetStart; targetIndex != targetEndExclusive; targetIndex += targetStep) {
|
||||
final char targetCharacter = targetCharacters[targetIndex + targetCharacterOffset];
|
||||
final int targetNeighbor = targetIndex + targetNeighborDelta;
|
||||
|
||||
final int deleteCandidate = this.costMatrix[sourceNeighbor][targetIndex] + this.deleteCost;
|
||||
final int insertCandidate = this.costMatrix[sourceIndex][targetNeighbor] + this.insertCost;
|
||||
final int replaceCandidate = this.costMatrix[sourceNeighbor][targetNeighbor] + this.replaceCost;
|
||||
final int matchCandidate = this.costMatrix[sourceNeighbor][targetNeighbor]
|
||||
+ (sourceCharacter == targetCharacter ? this.matchCost : MISMATCH_PENALTY);
|
||||
|
||||
int bestCost = matchCandidate;
|
||||
@@ -606,7 +933,8 @@ public final class PatchCommandEncoder {
|
||||
* @param targetLength target length
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String buildPatchCommand(final char[] targetCharacters, final int sourceLength, final int targetLength) {
|
||||
private String buildPatchCommandBackward(final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength) {
|
||||
final StringBuilder patchBuilder = new StringBuilder(sourceLength + targetLength);
|
||||
|
||||
char pendingDeletes = COUNT_SENTINEL;
|
||||
@@ -674,6 +1002,83 @@ public final class PatchCommandEncoder {
|
||||
return patchBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstructs compact patch command for forward traversal.
|
||||
*
|
||||
* @param targetCharacters target characters
|
||||
* @param sourceLength source length
|
||||
* @param targetLength target length
|
||||
* @return compact patch command
|
||||
*/
|
||||
private String buildPatchCommandForward(final char[] targetCharacters, final int sourceLength,
|
||||
final int targetLength) {
|
||||
final StringBuilder patchBuilder = new StringBuilder(sourceLength + targetLength);
|
||||
|
||||
char pendingDeletes = COUNT_SENTINEL;
|
||||
char pendingSkips = COUNT_SENTINEL;
|
||||
|
||||
int sourceIndex = 0;
|
||||
int targetIndex = 0;
|
||||
|
||||
while (sourceIndex != sourceLength || targetIndex != targetLength) {
|
||||
final Trace trace = this.traceMatrix[sourceIndex][targetIndex];
|
||||
|
||||
switch (trace) {
|
||||
case DELETE:
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
pendingDeletes++;
|
||||
sourceIndex++;
|
||||
break;
|
||||
|
||||
case INSERT:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
appendInstruction(patchBuilder, INSERT_OPCODE, targetCharacters[targetIndex]);
|
||||
targetIndex++;
|
||||
break;
|
||||
|
||||
case REPLACE:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
if (pendingSkips != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, SKIP_OPCODE, pendingSkips);
|
||||
pendingSkips = COUNT_SENTINEL;
|
||||
}
|
||||
appendInstruction(patchBuilder, REPLACE_OPCODE, targetCharacters[targetIndex]);
|
||||
sourceIndex++;
|
||||
targetIndex++;
|
||||
break;
|
||||
|
||||
case MATCH:
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
pendingDeletes = COUNT_SENTINEL;
|
||||
}
|
||||
pendingSkips++;
|
||||
sourceIndex++;
|
||||
targetIndex++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingDeletes != COUNT_SENTINEL) {
|
||||
appendInstruction(patchBuilder, DELETE_OPCODE, pendingDeletes);
|
||||
}
|
||||
|
||||
return patchBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends one serialized instruction to the patch command builder.
|
||||
*
|
||||
@@ -684,4 +1089,88 @@ public final class PatchCommandEncoder {
|
||||
private static void appendInstruction(final StringBuilder patchBuilder, final char opcode, final char argument) {
|
||||
patchBuilder.append(opcode).append(argument);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fluent builder for creating direction-specialized {@link PatchCommandEncoder}
|
||||
* instances.
|
||||
*/
|
||||
public static final class Builder {
|
||||
private WordTraversalDirection traversalDirection = WordTraversalDirection.BACKWARD;
|
||||
private int insertCost = 1;
|
||||
private int deleteCost = 1;
|
||||
private int replaceCost = 1;
|
||||
private int matchCost; // = 0
|
||||
|
||||
/**
|
||||
* Creates a builder initialized with the default Egothor-compatible cost model
|
||||
* and backward traversal.
|
||||
*/
|
||||
public Builder() {
|
||||
// Default values are assigned in field initializers.
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets traversal direction used by the created encoder.
|
||||
*
|
||||
* @param value traversal direction
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder traversalDirection(final WordTraversalDirection value) {
|
||||
this.traversalDirection = Objects.requireNonNull(value, "traversalDirection");
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of an insert operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder insertCost(final int value) {
|
||||
this.insertCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a delete operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder deleteCost(final int value) {
|
||||
this.deleteCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a replace operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder replaceCost(final int value) {
|
||||
this.replaceCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets cost of a match operation.
|
||||
*
|
||||
* @param value cost of the operation
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder matchCost(final int value) {
|
||||
this.matchCost = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a direction-specialized encoder instance.
|
||||
*
|
||||
* @return configured encoder
|
||||
*/
|
||||
public PatchCommandEncoder build() {
|
||||
return new PatchCommandEncoder(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ public final class StemmerKnowledgeExperiment {
|
||||
* Creates a new experiment harness.
|
||||
*/
|
||||
public StemmerKnowledgeExperiment() {
|
||||
this.patchCommandEncoder = new PatchCommandEncoder();
|
||||
this.patchCommandEncoder = PatchCommandEncoder.builder().build();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -132,6 +132,48 @@ public final class StemmerPatchTrieBinaryIO {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||
* at a filesystem path.
|
||||
*
|
||||
* @param path source file
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
return read(path).metadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie stored
|
||||
* at a filesystem path string.
|
||||
*
|
||||
* @param fileName source file name or path string
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final String fileName) throws IOException {
|
||||
Objects.requireNonNull(fileName, "fileName");
|
||||
return readMetadata(Path.of(fileName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only metadata from a GZip-compressed binary patch-command trie from an
|
||||
* input stream.
|
||||
*
|
||||
* @param inputStream source stream
|
||||
* @return deserialized trie metadata
|
||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||
* @throws IOException if reading or decompression fails
|
||||
*/
|
||||
public static TrieMetadata readMetadata(final InputStream inputStream) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
return read(inputStream).metadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a GZip-compressed binary patch-command trie to a filesystem path.
|
||||
*
|
||||
|
||||
@@ -267,6 +267,24 @@ public final class StemmerPatchTrieLoader {
|
||||
/**
|
||||
* Loads a bundled dictionary using explicit reduction settings.
|
||||
*
|
||||
* <p>
|
||||
* This overload applies the following implicit compilation defaults in addition
|
||||
* to the supplied {@code reductionSettings}:
|
||||
* </p>
|
||||
* <ul>
|
||||
* <li>traversal direction is derived from {@link Language#isRightToLeft()}
|
||||
* ({@link WordTraversalDirection#FORWARD} for right-to-left languages,
|
||||
* {@link WordTraversalDirection#BACKWARD} otherwise)</li>
|
||||
* <li>case processing mode is
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}</li>
|
||||
* <li>diacritic processing mode is {@link DiacriticProcessingMode#AS_IS}</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* The resolved settings are persisted into {@link TrieMetadata} of the
|
||||
* resulting trie.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -279,14 +297,40 @@ public final class StemmerPatchTrieLoader {
|
||||
final ReductionSettings reductionSettings) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
final TrieMetadata metadata = metadataForCompilation(traversalDirectionOf(language), reductionSettings,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
return load(language, storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bundled dictionary using explicit trie compilation metadata.
|
||||
*
|
||||
* <p>
|
||||
* All semantic compilation settings (reduction mode and thresholds, traversal
|
||||
* direction, case processing mode, and diacritic processing mode) are taken
|
||||
* from the supplied metadata object and are persisted unchanged in the
|
||||
* resulting trie.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the dictionary cannot be found or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Language language, final boolean storeOriginal,
|
||||
final TrieMetadata metadata) throws IOException {
|
||||
Objects.requireNonNull(language, "language");
|
||||
Objects.requireNonNull(metadata, "metadata");
|
||||
|
||||
final String resourcePath = language.resourcePath();
|
||||
|
||||
try (InputStream inputStream = openBundledResource(resourcePath);
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
return load(reader, resourcePath, storeOriginal, reductionSettings, traversalDirectionOf(language),
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
return load(reader, resourcePath, storeOriginal, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -294,6 +338,14 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a bundled dictionary using default settings for the supplied reduction
|
||||
* mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload is equivalent to calling
|
||||
* {@link #load(Language, boolean, ReductionSettings)} with
|
||||
* {@link ReductionSettings#withDefaults(ReductionMode)} and therefore uses the
|
||||
* same implicit defaults for traversal direction, case processing mode, and
|
||||
* diacritic processing mode.
|
||||
* </p>
|
||||
*
|
||||
* @param language bundled language dictionary
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -311,6 +363,14 @@ public final class StemmerPatchTrieLoader {
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings.
|
||||
*
|
||||
* <p>
|
||||
* This overload applies historical Egothor-compatible implicit defaults:
|
||||
* {@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT}, and
|
||||
* {@link DiacriticProcessingMode#AS_IS}. These settings are persisted in
|
||||
* resulting trie metadata.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -322,13 +382,19 @@ public final class StemmerPatchTrieLoader {
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings) throws IOException {
|
||||
return load(path, storeOriginal, reductionSettings, WordTraversalDirection.BACKWARD,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings
|
||||
* and explicit traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* Implicit defaults still apply for unspecified dimensions:
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT} and
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
@@ -343,13 +409,18 @@ public final class StemmerPatchTrieLoader {
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection)
|
||||
throws IOException {
|
||||
return load(path, storeOriginal, reductionSettings, traversalDirection,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT, DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings,
|
||||
* explicit traversal direction, and explicit case processing mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload still defaults diacritic processing to
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
@@ -364,16 +435,65 @@ public final class StemmerPatchTrieLoader {
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode) throws IOException {
|
||||
return load(path, storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit reduction settings,
|
||||
* traversal direction, case processing mode, and diacritic processing mode.
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted
|
||||
* using the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys
|
||||
* and patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary
|
||||
* parsing
|
||||
* @param diacriticProcessingMode diacritic processing mode used during
|
||||
* dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
final TrieMetadata metadata = metadataForCompilation(traversalDirection, reductionSettings, caseProcessingMode,
|
||||
diacriticProcessingMode);
|
||||
return load(path, storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path using explicit trie compilation
|
||||
* metadata.
|
||||
*
|
||||
* <p>
|
||||
* The supplied metadata is the authoritative source of trie compilation
|
||||
* semantics. Callers should ensure metadata matches how they expect to query
|
||||
* the trie (for example, with or without lowercasing or diacritic stripping).
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final Path path, final boolean storeOriginal, final TrieMetadata metadata)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
Objects.requireNonNull(metadata, "metadata");
|
||||
|
||||
try (InputStream inputStream = openDictionaryInputStream(path);
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
|
||||
return load(reader, path.toAbsolutePath().toString(), storeOriginal, reductionSettings, traversalDirection,
|
||||
caseProcessingMode);
|
||||
return load(reader, path.toAbsolutePath().toString(), storeOriginal, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,6 +501,15 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path using default settings for the
|
||||
* supplied reduction mode.
|
||||
*
|
||||
* <p>
|
||||
* This overload is equivalent to calling
|
||||
* {@link #load(Path, boolean, ReductionSettings)} with
|
||||
* {@link ReductionSettings#withDefaults(ReductionMode)} and therefore uses
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param path path to the dictionary file
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -399,6 +528,13 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as {@link #load(Path, boolean, ReductionSettings)} including
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -417,6 +553,14 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings and explicit traversal direction.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as
|
||||
* {@link #load(Path, boolean, ReductionSettings, WordTraversalDirection)}.
|
||||
* Implicit defaults remain
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT} and
|
||||
* {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
@@ -439,6 +583,12 @@ public final class StemmerPatchTrieLoader {
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings, explicit traversal direction, and explicit case processing mode.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as
|
||||
* {@link #load(Path, boolean, ReductionSettings, WordTraversalDirection, CaseProcessingMode)}.
|
||||
* Implicit default remains {@link DiacriticProcessingMode#AS_IS}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using
|
||||
* the canonical no-op patch command
|
||||
@@ -454,13 +604,71 @@ public final class StemmerPatchTrieLoader {
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, caseProcessingMode);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
DiacriticProcessingMode.AS_IS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit reduction
|
||||
* settings, explicit traversal direction, explicit case processing mode, and
|
||||
* explicit diacritic processing mode.
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted
|
||||
* using the canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param traversalDirection traversal direction used for both trie keys
|
||||
* and patch commands
|
||||
* @param caseProcessingMode case processing mode used during dictionary
|
||||
* parsing
|
||||
* @param diacriticProcessingMode diacritic processing mode used during
|
||||
* dictionary parsing
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final ReductionSettings reductionSettings, final WordTraversalDirection traversalDirection,
|
||||
final CaseProcessingMode caseProcessingMode, final DiacriticProcessingMode diacriticProcessingMode)
|
||||
throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, reductionSettings, traversalDirection, caseProcessingMode,
|
||||
diacriticProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using explicit trie
|
||||
* compilation metadata.
|
||||
*
|
||||
* <p>
|
||||
* Same semantics as {@link #load(Path, boolean, TrieMetadata)}.
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param metadata trie metadata describing the compilation configuration
|
||||
* @return compiled patch-command trie
|
||||
* @throws NullPointerException if any argument is {@code null}
|
||||
* @throws IOException if the file cannot be opened or read
|
||||
*/
|
||||
public static FrequencyTrie<String> load(final String fileName, final boolean storeOriginal,
|
||||
final TrieMetadata metadata) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return load(Path.of(fileName), storeOriginal, metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a dictionary from a filesystem path string using default settings for
|
||||
* the supplied reduction mode.
|
||||
*
|
||||
* <p>
|
||||
* Equivalent to {@link #load(Path, boolean, ReductionMode)} and therefore uses
|
||||
* implicit defaults ({@link WordTraversalDirection#BACKWARD},
|
||||
* {@link CaseProcessingMode#LOWERCASE_WITH_LOCALE_ROOT},
|
||||
* {@link DiacriticProcessingMode#AS_IS}).
|
||||
* </p>
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
@@ -482,21 +690,21 @@ public final class StemmerPatchTrieLoader {
|
||||
* @param sourceDescription logical source description used for diagnostics
|
||||
* @param storeOriginal whether the stem itself should be inserted using the
|
||||
* canonical no-op patch command
|
||||
* @param reductionSettings reduction settings
|
||||
* @param metadata trie metadata used to drive all compilation settings
|
||||
* @return compiled patch-command trie
|
||||
* @throws IOException if parsing fails
|
||||
*/
|
||||
private static FrequencyTrie<String> load(final BufferedReader reader, final String sourceDescription,
|
||||
final boolean storeOriginal, final ReductionSettings reductionSettings,
|
||||
final WordTraversalDirection traversalDirection, final CaseProcessingMode caseProcessingMode)
|
||||
throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new, reductionSettings,
|
||||
traversalDirection, caseProcessingMode);
|
||||
final PatchCommandEncoder patchCommandEncoder = new PatchCommandEncoder(traversalDirection);
|
||||
final boolean storeOriginal, final TrieMetadata metadata) throws IOException {
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(String[]::new,
|
||||
metadata.reductionSettings(), metadata.traversalDirection(), metadata.caseProcessingMode(),
|
||||
metadata.diacriticProcessingMode());
|
||||
final PatchCommandEncoder patchCommandEncoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(metadata.traversalDirection()).build();
|
||||
final int[] insertedMappings = new int[1];
|
||||
|
||||
final StemmerDictionaryParser.ParseStatistics statistics = StemmerDictionaryParser.parse(reader,
|
||||
sourceDescription, caseProcessingMode, (stem, variants, lineNumber) -> {
|
||||
sourceDescription, metadata.caseProcessingMode(), (stem, variants, lineNumber) -> {
|
||||
if (storeOriginal) {
|
||||
builder.put(stem, NOOP_PATCH_COMMAND);
|
||||
insertedMappings[0]++;
|
||||
@@ -512,14 +720,25 @@ public final class StemmerPatchTrieLoader {
|
||||
|
||||
if (LOGGER.isLoggable(Level.FINE)) {
|
||||
LOGGER.log(Level.FINE,
|
||||
"Loaded stemmer dictionary from {0}; insertedMappings={1}, lines={2}, entries={3}, ignoredLines={4}, traversalDirection={5}.",
|
||||
"Loaded stemmer dictionary from {0}; insertedMappings={1}, lines={2}, entries={3}, ignoredLines={4}, metadata={5}.",
|
||||
new Object[] { sourceDescription, insertedMappings[0], statistics.lineCount(),
|
||||
statistics.entryCount(), statistics.ignoredLineCount(), traversalDirection });
|
||||
statistics.entryCount(), statistics.ignoredLineCount(), metadata.toTextBlock() });
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static TrieMetadata metadataForCompilation(final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings, final CaseProcessingMode caseProcessingMode,
|
||||
final DiacriticProcessingMode diacriticProcessingMode) {
|
||||
Objects.requireNonNull(traversalDirection, "traversalDirection");
|
||||
Objects.requireNonNull(reductionSettings, "reductionSettings");
|
||||
Objects.requireNonNull(caseProcessingMode, "caseProcessingMode");
|
||||
Objects.requireNonNull(diacriticProcessingMode, "diacriticProcessingMode");
|
||||
return TrieMetadata.forCompilation(traversalDirection, reductionSettings, diacriticProcessingMode,
|
||||
caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the traversal direction implied by a bundled language definition.
|
||||
*
|
||||
@@ -572,6 +791,50 @@ public final class StemmerPatchTrieLoader {
|
||||
return StemmerPatchTrieBinaryIO.read(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie file.
|
||||
*
|
||||
* @param path path to the compressed binary trie file
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code path} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final Path path) throws IOException {
|
||||
Objects.requireNonNull(path, "path");
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie file.
|
||||
*
|
||||
* @param fileName file name or path string
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code fileName} is {@code null}
|
||||
* @throws IOException if the file cannot be opened, decompressed, or
|
||||
* read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final String fileName) throws IOException {
|
||||
Objects.requireNonNull(fileName, FILENAME_REQUIRED);
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads only persisted metadata from a GZip-compressed binary patch-command
|
||||
* trie stream.
|
||||
*
|
||||
* @param inputStream source input stream
|
||||
* @return persisted trie metadata
|
||||
* @throws NullPointerException if {@code inputStream} is {@code null}
|
||||
* @throws IOException if the stream cannot be decompressed or read
|
||||
*/
|
||||
public static TrieMetadata loadBinaryMetadata(final InputStream inputStream) throws IOException {
|
||||
Objects.requireNonNull(inputStream, "inputStream");
|
||||
return StemmerPatchTrieBinaryIO.readMetadata(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a compiled patch-command trie as a GZip-compressed binary file.
|
||||
*
|
||||
|
||||
@@ -105,6 +105,23 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates metadata for a newly compiled trie using the currently persisted
|
||||
* binary stream format version.
|
||||
*
|
||||
* @param traversalDirection logical key traversal direction
|
||||
* @param reductionSettings reduction settings used during compilation
|
||||
* @param diacriticProcessingMode diacritic processing strategy
|
||||
* @param caseProcessingMode case processing strategy
|
||||
* @return metadata aligned with the current persisted stream format
|
||||
*/
|
||||
public static TrieMetadata forCompilation(final WordTraversalDirection traversalDirection,
|
||||
final ReductionSettings reductionSettings, final DiacriticProcessingMode diacriticProcessingMode,
|
||||
final CaseProcessingMode caseProcessingMode) {
|
||||
return new TrieMetadata(FrequencyTrie.currentFormatVersion(), traversalDirection, reductionSettings,
|
||||
diacriticProcessingMode, caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates metadata compatible with a legacy artifact version that did not store
|
||||
* the full configuration explicitly.
|
||||
@@ -200,6 +217,14 @@ public record TrieMetadata(int formatVersion, WordTraversalDirection traversalDi
|
||||
diacriticProcessingMode, caseProcessingMode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a required metadata entry from a parsed text block.
|
||||
*
|
||||
* @param entries parsed metadata entries
|
||||
* @param key required entry key
|
||||
* @return non-blank entry value
|
||||
* @throws IllegalArgumentException if the entry is absent or blank
|
||||
*/
|
||||
private static String requireEntry(final Map<String, String> entries, final String key) {
|
||||
final String value = entries.get(key);
|
||||
if (value == null || value.isBlank()) {
|
||||
|
||||
@@ -60,11 +60,23 @@ import java.util.Objects;
|
||||
this.childSignature = childSignature;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(this.edge, this.childSignature);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -52,6 +52,11 @@ import java.util.Objects;
|
||||
@SuppressWarnings("PMD.DataClass")
|
||||
public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[] orderedValues, int... orderedCounts) {
|
||||
|
||||
/**
|
||||
* Number of child edges where linear scan is cheaper than binary search.
|
||||
*/
|
||||
private static final int LINEAR_CHILD_COUNT_THRESHOLD = 4;
|
||||
|
||||
/**
|
||||
* Creates one validated compiled node.
|
||||
*
|
||||
@@ -140,6 +145,19 @@ public record CompiledNode<V>(char[] edgeLabels, CompiledNode<V>[] children, V[]
|
||||
* @return child node, or {@code null} if absent
|
||||
*/
|
||||
public CompiledNode<V> findChild(final char edge) {
|
||||
final int childCount = this.edgeLabels.length;
|
||||
if (childCount == 0) {
|
||||
return null;
|
||||
}
|
||||
if (childCount <= LINEAR_CHILD_COUNT_THRESHOLD) {
|
||||
for (int index = 0; index < childCount; index++) {
|
||||
if (this.edgeLabels[index] == edge) {
|
||||
return this.children[index];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
final int index = Arrays.binarySearch(this.edgeLabels, edge);
|
||||
if (index < 0) {
|
||||
return null;
|
||||
|
||||
@@ -53,11 +53,23 @@ import java.util.Objects;
|
||||
this.dominantValue = dominantValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hashCode(this.dominantValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -65,11 +65,23 @@ import java.util.List;
|
||||
Collections.unmodifiableList(Arrays.asList(Arrays.copyOf(orderedValues, orderedValues.length))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return this.orderedValues.hashCode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
@@ -67,11 +67,23 @@ import java.util.Set;
|
||||
return new UnorderedLocalDescriptor(Collections.unmodifiableSet(distinct));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a hash code consistent with descriptor equality.
|
||||
*
|
||||
* @return descriptor hash code
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return this.distinctValues.hashCode();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares this descriptor with another object.
|
||||
*
|
||||
* @param other object to compare with
|
||||
* @return {@code true} when both descriptors represent the same semantic
|
||||
* reduction identity
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(final Object other) {
|
||||
if (this == other) {
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -63,7 +63,7 @@ class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||
@Label("encode followed by apply should reconstruct the target word")
|
||||
void encodeFollowedByApplyShouldReconstructTheTargetWord(@ForAll("words") final String source,
|
||||
@ForAll("words") final String target) {
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
final String patch = encoder.encode(source, target);
|
||||
|
||||
assertNotNull(patch, "patch generation must succeed for non-null inputs.");
|
||||
@@ -82,10 +82,10 @@ class PatchCommandEncoderProperties extends PropertyBasedTestSupport {
|
||||
@Label("encode should be deterministic for one source-target pair")
|
||||
void encodeShouldBeDeterministicForOneSourceTargetPair(@ForAll("words") final String source,
|
||||
@ForAll("words") final String target) {
|
||||
final PatchCommandEncoder sharedEncoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder sharedEncoder = PatchCommandEncoder.builder().build();
|
||||
final String first = sharedEncoder.encode(source, target);
|
||||
final String second = sharedEncoder.encode(source, target);
|
||||
final String fresh = new PatchCommandEncoder().encode(source, target);
|
||||
final String fresh = PatchCommandEncoder.builder().build().encode(source, target);
|
||||
|
||||
assertEquals(first, second, "one encoder instance must produce stable output.");
|
||||
assertEquals(first, fresh, "fresh encoder instances must produce the same patch output.");
|
||||
|
||||
@@ -250,12 +250,28 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("creates encoder with default cost model")
|
||||
void shouldCreateEncoderWithDefaultCostModel() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertNotNull(encoder);
|
||||
assertEquals("teach", PatchCommandEncoder.apply("teacher", encoder.encode("teacher", "teach")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies fluent builder construction with explicit forward traversal.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("builds direction-specialized encoder via builder")
|
||||
void shouldBuildDirectionSpecializedEncoderViaBuilder() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
|
||||
String patch = encoder.encode("running", "run");
|
||||
|
||||
assertAll(() -> assertNotNull(encoder), () -> assertNotNull(patch),
|
||||
() -> assertEquals("run", encoder.applyWithConfiguredDirection("running", patch)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that a negative insert cost is rejected.
|
||||
*/
|
||||
@@ -263,7 +279,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative insert cost")
|
||||
void shouldRejectNegativeInsertCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(-1, 1, 1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(-1).deleteCost(1).replaceCost(1).matchCost(0).build());
|
||||
|
||||
assertEquals("insertCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -275,7 +291,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative delete cost")
|
||||
void shouldRejectNegativeDeleteCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, -1, 1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(-1).replaceCost(1).matchCost(0).build());
|
||||
|
||||
assertEquals("deleteCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -287,7 +303,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative replace cost")
|
||||
void shouldRejectNegativeReplaceCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, 1, -1, 0));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(-1).matchCost(0).build());
|
||||
|
||||
assertEquals("replaceCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -299,7 +315,7 @@ class PatchCommandEncoderTest {
|
||||
@DisplayName("rejects negative match cost")
|
||||
void shouldRejectNegativeMatchCost() {
|
||||
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class,
|
||||
() -> new PatchCommandEncoder(1, 1, 1, -1));
|
||||
() -> PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(1).matchCost(-1).build());
|
||||
|
||||
assertEquals("matchCost must be non-negative.", exception.getMessage());
|
||||
}
|
||||
@@ -320,7 +336,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("does not emit trailing SKIP instructions into patch command")
|
||||
void shouldNotEmitTrailingSkipInstructionsIntoPatchCommand() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("abcd", "ab");
|
||||
|
||||
@@ -335,7 +351,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns null when source is null")
|
||||
void shouldReturnNullWhenSourceIsNull() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode(null, "target");
|
||||
|
||||
@@ -348,7 +364,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns null when target is null")
|
||||
void shouldReturnNullWhenTargetIsNull() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("source", null);
|
||||
|
||||
@@ -361,7 +377,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns canonical NOOP patch for equal words")
|
||||
void shouldReturnCanonicalNoopPatchForEqualWords() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teacher");
|
||||
|
||||
@@ -375,7 +391,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("returns canonical NOOP patch for equal empty words")
|
||||
void shouldReturnCanonicalNoopPatchForEqualEmptyWords() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("", "");
|
||||
|
||||
@@ -394,7 +410,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideRoundTripPairs")
|
||||
@DisplayName("produces patches that reconstruct the target")
|
||||
void shouldReconstructTargetForRoundTripPairs(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode(source, target);
|
||||
String reconstructed = PatchCommandEncoder.apply(source, patch);
|
||||
@@ -414,7 +430,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("remains correct when reused across different input sizes")
|
||||
void shouldRemainCorrectWhenReusedAcrossDifferentInputSizes() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals("transformation",
|
||||
@@ -430,7 +446,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("supports custom operation costs")
|
||||
void shouldSupportCustomOperationCosts() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder(1, 1, 2, 0);
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().insertCost(1).deleteCost(1).replaceCost(2).matchCost(0).build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teach");
|
||||
String reconstructed = PatchCommandEncoder.apply("teacher", patch);
|
||||
@@ -489,6 +505,36 @@ class PatchCommandEncoderTest {
|
||||
assertSame(source, PatchCommandEncoder.apply(source, PatchCommandEncoder.NOOP_PATCH));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that instance-level application follows encoder traversal
|
||||
* direction.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("applies patch via instance-level direction-specialized fast path")
|
||||
void shouldApplyPatchViaInstanceLevelDirectionSpecializedFastPath() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
|
||||
String patch = encoder.encode("transformation", "transform");
|
||||
|
||||
assertEquals("transform", encoder.applyWithConfiguredDirection("transformation", patch));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies dedicated forward traversal encode/apply round trip.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("reconstructs target with forward traversal encoder and static apply")
|
||||
void shouldReconstructTargetWithForwardTraversalEncoderAndStaticApply() {
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder()
|
||||
.traversalDirection(WordTraversalDirection.FORWARD)
|
||||
.build();
|
||||
String patch = encoder.encode("cities", "city");
|
||||
|
||||
assertEquals("city", PatchCommandEncoder.apply("cities", patch, WordTraversalDirection.FORWARD));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies explicit patch application cases.
|
||||
*
|
||||
@@ -560,7 +606,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles deletion-heavy suffix stripping")
|
||||
void shouldHandleDeletionHeavySuffixStripping() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("teacher", "teach");
|
||||
|
||||
@@ -573,7 +619,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles plural to singular transformation")
|
||||
void shouldHandlePluralToSingularTransformation() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("cities", "city");
|
||||
|
||||
@@ -586,7 +632,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles derivational reduction to a shorter stem")
|
||||
void shouldHandleDerivationalReductionToShorterStem() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("stemming", "stem");
|
||||
|
||||
@@ -599,7 +645,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles single-character replacement")
|
||||
void shouldHandleSingleCharacterReplacement() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String patch = encoder.encode("a", "z");
|
||||
|
||||
@@ -626,7 +672,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||
@DisplayName("reconstructs reversed targets from reversed sources")
|
||||
void shouldReconstructReversedTargetsFromReversedSources(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String reversedSource = reverse(source);
|
||||
String reversedTarget = reverse(target);
|
||||
@@ -649,7 +695,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("handles mirrored stemming transformations")
|
||||
void shouldHandleMirroredStemmingTransformations() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(reverse("teach"),
|
||||
@@ -671,7 +717,7 @@ class PatchCommandEncoderTest {
|
||||
@Test
|
||||
@DisplayName("remains correct when reused on reversed words of different sizes")
|
||||
void shouldRemainCorrectWhenReusedOnReversedWordsOfDifferentSizes() {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
assertAll(
|
||||
() -> assertEquals(reverse("transformation"),
|
||||
@@ -699,7 +745,7 @@ class PatchCommandEncoderTest {
|
||||
@MethodSource("org.egothor.stemmer.PatchCommandEncoderTest#provideReversedRoundTripPairs")
|
||||
@DisplayName("preserves correctness under mirrored input orientation")
|
||||
void shouldPreserveCorrectnessUnderMirroredInputOrientation(int caseId, String source, String target) {
|
||||
PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
String normalPatch = encoder.encode(source, target);
|
||||
String normalResult = PatchCommandEncoder.apply(source, normalPatch);
|
||||
|
||||
@@ -151,7 +151,7 @@ abstract class PropertyBasedTestSupport {
|
||||
Objects.requireNonNull(reductionMode, "reductionMode");
|
||||
|
||||
final FrequencyTrie.Builder<String> builder = new FrequencyTrie.Builder<>(STRING_ARRAY_FACTORY, reductionMode);
|
||||
final PatchCommandEncoder encoder = new PatchCommandEncoder();
|
||||
final PatchCommandEncoder encoder = PatchCommandEncoder.builder().build();
|
||||
|
||||
for (StemmerEntry entry : scenario.entries()) {
|
||||
if (storeOriginal) {
|
||||
|
||||
@@ -158,7 +158,7 @@ final class StemmerPatchTrieLoaderTest {
|
||||
static Stream<Arguments> nullContractCases() {
|
||||
final ReductionSettings settings = ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE);
|
||||
final FrequencyTrie<String> trie = new FrequencyTrie.Builder<String>(String[]::new, settings)
|
||||
.put("running", new PatchCommandEncoder().encode("running", "run")).build();
|
||||
.put("running", PatchCommandEncoder.builder().build().encode("running", "run")).build();
|
||||
|
||||
return Stream.of(
|
||||
Arguments.of("01-load-language-settings",
|
||||
@@ -222,7 +222,26 @@ final class StemmerPatchTrieLoaderTest {
|
||||
"trie"),
|
||||
Arguments.of("19-save-binary-null-string",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.saveBinary(trie, (String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED));
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("20-load-language-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(StemmerPatchTrieLoader.Language.US_UK,
|
||||
true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("21-load-path-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath(), true, (TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("22-load-string-null-metadata",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.load(tempPath().toString(), true,
|
||||
(TrieMetadata) null),
|
||||
"metadata"),
|
||||
Arguments.of("23-load-binary-metadata-path-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((Path) null), "path"),
|
||||
Arguments.of("24-load-binary-metadata-string-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((String) null),
|
||||
StemmerPatchTrieLoader.FILENAME_REQUIRED),
|
||||
Arguments.of("25-load-binary-metadata-stream-null",
|
||||
(ExecutableOperation) () -> StemmerPatchTrieLoader.loadBinaryMetadata((InputStream) null),
|
||||
"inputStream"));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -327,6 +346,31 @@ final class StemmerPatchTrieLoaderTest {
|
||||
"run");
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that metadata-driven loading keeps all configuration dimensions in
|
||||
* one explicit object and applies them during compilation.
|
||||
*
|
||||
* @throws IOException if the test file cannot be written or read
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Metadata overload must drive case and diacritic normalization")
|
||||
void shouldLoadUsingExplicitMetadataConfiguration() throws IOException {
|
||||
final Path dictionaryFile = writeDictionary("""
|
||||
mÁma mamA mámě
|
||||
""");
|
||||
final TrieMetadata metadata = TrieMetadata.forCompilation(WordTraversalDirection.BACKWARD,
|
||||
ReductionSettings.withDefaults(DEFAULT_REDUCTION_MODE), DiacriticProcessingMode.REMOVE,
|
||||
CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT);
|
||||
|
||||
final FrequencyTrie<String> trie = StemmerPatchTrieLoader.load(dictionaryFile, true, metadata);
|
||||
|
||||
assertAll(() -> assertEquals(DiacriticProcessingMode.REMOVE, trie.metadata().diacriticProcessingMode()),
|
||||
() -> assertEquals(CaseProcessingMode.LOWERCASE_WITH_LOCALE_ROOT,
|
||||
trie.metadata().caseProcessingMode()),
|
||||
() -> assertNotNull(trie.get("MÁMĚ")),
|
||||
() -> assertNotNull(trie.get("mame")));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the loader honors {@code storeOriginal=true} by inserting the
|
||||
* canonical no-op patch for the stem itself.
|
||||
@@ -457,6 +501,15 @@ final class StemmerPatchTrieLoaderTest {
|
||||
assertTriePatchSemanticsEqual(original, fromString, "run", "running", "runner", "cities", "studying");
|
||||
assertTriePatchSemanticsEqual(original, fromStream, "run", "running", "runner", "cities", "studying");
|
||||
}
|
||||
|
||||
final TrieMetadata metadataFromPath = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile);
|
||||
final TrieMetadata metadataFromString = StemmerPatchTrieLoader.loadBinaryMetadata(binaryFile.toString());
|
||||
try (InputStream metadataInputStream = new ByteArrayInputStream(binaryBytes)) {
|
||||
final TrieMetadata metadataFromStream = StemmerPatchTrieLoader.loadBinaryMetadata(metadataInputStream);
|
||||
assertAll(() -> assertEquals(original.metadata(), metadataFromPath),
|
||||
() -> assertEquals(original.metadata(), metadataFromString),
|
||||
() -> assertEquals(original.metadata(), metadataFromStream));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user