Skip to content

Commit bf769ff

Browse files
committed
feat(core): add caseless letters and symbols Unicode support
- Added `caselessLettersUnicode()` to explicitly support the `\p{Lo}` category, enabling safe validation of scripts without uppercase/lowercase distinction (e.g., Kanji, Arabic). - Added `symbolsUnicode()` to support the `\p{S}` category for currencies, mathematical operators, and dingbats. - Added comprehensive cookbook tests demonstrating real-world usage for international compound names and symbol parsing.
1 parent 8043b6e commit bf769ff

6 files changed

Lines changed: 158 additions & 1 deletion

File tree

sift-core/src/main/java/com/mirkoddd/sift/core/BaseType.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,22 @@ public C lowerCaseLettersUnicode() {
190190
return getCharacterClassConnector(next);
191191
}
192192

193+
/** {@inheritDoc} */
194+
@Override
195+
public C caselessLettersUnicode() {
196+
PatternAssembler next = assembler.copy();
197+
next.addClassRange(RegexSyntax.UNICODE_LETTERS_CASELESS);
198+
return getCharacterClassConnector(next);
199+
}
200+
201+
/** {@inheritDoc} */
202+
@Override
203+
public C symbolsUnicode() {
204+
PatternAssembler next = assembler.copy();
205+
next.addClassRange(RegexSyntax.UNICODE_SYMBOLS);
206+
return getCharacterClassConnector(next);
207+
}
208+
193209
/** {@inheritDoc} */
194210
@Override
195211
public C alphanumeric() {

sift-core/src/main/java/com/mirkoddd/sift/core/RegexSyntax.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ final class RegexSyntax {
7373
static final String NON_UNICODE_LETTERS = "\\P{L}";
7474
static final String UNICODE_LETTERS_UPPERCASE_ONLY = "\\p{Lu}";
7575
static final String UNICODE_LETTERS_LOWERCASE_ONLY = "\\p{Ll}";
76+
public static final String UNICODE_LETTERS_CASELESS = "\\p{Lo}";
77+
public static final String UNICODE_SYMBOLS = "\\p{S}";
7678
static final String UNICODE_ALPHANUMERIC = "\\p{L}\\p{Nd}"; // Stripped outer brackets
7779
static final String NON_UNICODE_ALPHANUMERIC = "^\\p{L}\\p{Nd}"; // Stripped outer brackets
7880
static final String UNICODE_WORD_CHARACTERS = "\\p{L}\\p{Nd}_"; // Stripped outer brackets

sift-core/src/main/java/com/mirkoddd/sift/core/SiftQuantifier.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,17 @@ public Connector<Ctx> backreference(NamedCapture group) {
154154
@Override public CharacterConnector<Ctx> nonLettersUnicode() { return exactly(1).nonLettersUnicode(); }
155155
@Override public CharacterConnector<Ctx> upperCaseLettersUnicode() { return exactly(1).upperCaseLettersUnicode(); }
156156
@Override public CharacterConnector<Ctx> lowerCaseLettersUnicode() { return exactly(1).lowerCaseLettersUnicode(); }
157+
158+
@Override
159+
public CharacterConnector<Ctx> caselessLettersUnicode() {
160+
return exactly(1).caselessLettersUnicode();
161+
}
162+
163+
@Override
164+
public CharacterConnector<Ctx> symbolsUnicode() {
165+
return exactly(1).symbolsUnicode();
166+
}
167+
157168
@Override public CharacterConnector<Ctx> alphanumeric() { return exactly(1).alphanumeric(); }
158169
@Override public CharacterConnector<Ctx> nonAlphanumeric() { return exactly(1).nonAlphanumeric(); }
159170
@Override public CharacterConnector<Ctx> alphanumericUnicode() { return exactly(1).alphanumericUnicode(); }

sift-core/src/main/java/com/mirkoddd/sift/core/dsl/Type.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,24 @@ public interface Type<Ctx extends SiftContext, T extends Connector<Ctx>, C exten
184184
*/
185185
C lowerCaseLettersUnicode();
186186

187+
/**
188+
* Matches any Unicode caseless letter (e.g., Kanji, Arabic, Hebrew).
189+
* <p>
190+
* Equivalent to the regex {@code \p{Lo}} (Letter, Other).
191+
*
192+
* @return The specialized character class step to allow further class modifications.
193+
*/
194+
C caselessLettersUnicode();
195+
196+
/**
197+
* Matches any Unicode symbol character (e.g., currency signs, mathematical operators).
198+
* <p>
199+
* Equivalent to the regex {@code \p{S}}.
200+
*
201+
* @return The specialized character class step to allow further class modifications.
202+
*/
203+
C symbolsUnicode();
204+
187205
/**
188206
* Matches any ASCII alphanumeric character (letters and digits).
189207
* <p>

sift-core/src/test/java/com/mirkoddd/sift/core/ImplicitQuantifierCoverageTest.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ void verifyLetterVariants() {
115115
Sift.fromStart().upperCaseLettersUnicode().shake(),
116116
Sift.fromStart().exactly(1).upperCaseLettersUnicode().shake()
117117
);
118+
assertImplicitEqualsExplicit(
119+
Sift.fromStart().caselessLettersUnicode().shake(),
120+
Sift.fromStart().exactly(1).caselessLettersUnicode().shake()
121+
);
118122
}
119123

120124
@Test
@@ -201,6 +205,10 @@ void verifyExtendedUtilityClasses() {
201205
Sift.fromStart().blankUnicode().shake(),
202206
Sift.fromStart().exactly(1).blankUnicode().shake()
203207
);
208+
assertImplicitEqualsExplicit(
209+
Sift.fromStart().symbolsUnicode().shake(),
210+
Sift.fromStart().exactly(1).symbolsUnicode().shake()
211+
);
204212
}
205213

206214
@Test

sift-core/src/test/java/com/mirkoddd/sift/core/SiftCookbookTest.java

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
import static com.mirkoddd.sift.core.Sift.fromWordBoundary;
2323
import static com.mirkoddd.sift.core.Sift.oneOrMore;
2424
import static com.mirkoddd.sift.core.Sift.optional;
25+
import static com.mirkoddd.sift.core.Sift.zeroOrMore;
2526
import static com.mirkoddd.sift.core.SiftPatterns.anyOf;
2627
import static com.mirkoddd.sift.core.SiftPatterns.literal;
27-
import static com.mirkoddd.sift.core.SiftPatterns.negativeLookahead;
2828
import static com.mirkoddd.sift.core.SiftPatterns.positiveLookahead;
2929

3030
import org.junit.jupiter.api.DisplayName;
@@ -465,4 +465,106 @@ void extractHydroxidesFromText() {
465465

466466
assertEquals(expected, extractedMolecules, "Extraction failed: Regex matched incorrect patterns or missed valid ones");
467467
}
468+
469+
@Test
470+
@DisplayName("Should validate international compound names with distinct rules for cased and caseless scripts")
471+
void testInternationalCompoundNames() {
472+
// --- RULE A: CASED NAMES (Western/Latin, Cyrillic, Greek, etc.) ---
473+
474+
// 1. Define the starting capital letter
475+
CharacterConnector<Fragment> upperCaseLettersUnicode = exactly(1).upperCaseLettersUnicode();
476+
477+
// 2. Define a single word: a capital letter followed by up to 49 lowercase letters.
478+
// Using withoutBacktracking() to optimize performance and prevent catastrophic backtracking.
479+
SiftPattern<Fragment> casedWord = upperCaseLettersUnicode
480+
.followedBy(between(0, 49).lowerCaseLettersUnicode().withoutBacktracking());
481+
482+
// 3. Define allowed separators for compound names (space or hyphen)
483+
SiftPattern<Fragment> nameSeparators = anyOf(
484+
exactly(1).character(' '),
485+
exactly(1).character('-')
486+
);
487+
488+
// 4. Define an extension: a separator followed by another properly cased word
489+
SiftPattern<Fragment> casedExtension = exactly(1).of(nameSeparators)
490+
.followedBy(casedWord);
491+
492+
// 5. Assemble the full cased name: a base word followed by any number of valid extensions
493+
Connector<Fragment> casedName = Sift.fromAnywhere().of(casedWord)
494+
.followedBy(zeroOrMore().of(casedExtension));
495+
496+
// --- RULE B: CASELESS NAMES (Kanji, Arabic, Hebrew, etc.) ---
497+
498+
// 1. Define a single caseless word (length 1 to 50)
499+
SiftPattern<Fragment> caselessWord = between(1, 50).caselessLettersUnicode().withoutBacktracking();
500+
501+
// 2. Define an extension: a space followed by another caseless word (hyphens are rare here)
502+
SiftPattern<Fragment> caselessExtension = exactly(1).character(' ')
503+
.followedBy(caselessWord);
504+
505+
// 3. Assemble the full caseless name
506+
Connector<Fragment> caselessName = Sift.fromAnywhere().of(caselessWord)
507+
.followedBy(zeroOrMore().of(caselessExtension));
508+
509+
// --- COMBINED CONDITIONAL LOGIC ---
510+
511+
// If the name starts with an uppercase letter, apply the strict cased rules.
512+
// Otherwise, assume it's a caseless script and apply the caseless rules.
513+
SiftPattern<Fragment> name = SiftPatterns.ifFollowedBy(upperCaseLettersUnicode)
514+
.thenUse(casedName)
515+
.otherwiseUse(caselessName);
516+
517+
// Anchor to start and end to ensure the entire string is just the name
518+
String regex = Sift.fromStart()
519+
.of(name)
520+
.andNothingElse()
521+
.shake();
522+
523+
// --- TESTS ---
524+
assertMatches(regex, "桜"); // Single caseless word (Kanji) - PASS
525+
assertMatches(regex, "Günther"); // Single cased word (Latin with umlaut) - PASS
526+
assertMatches(regex, "Gian Maria"); // Compound cased name with space - PASS
527+
assertMatches(regex, "Jean-Luc"); // Compound cased name with hyphen - PASS
528+
529+
// Invalid formats rejected by strict structural rules
530+
assertDoesNotMatch(regex, "Gian Maria"); // Fails: Double space not allowed (Security!)
531+
assertDoesNotMatch(regex, "Gian maria"); // Fails: Second word missing capital letter
532+
assertDoesNotMatch(regex, " Gian"); // Fails: Cannot start with a space
533+
assertDoesNotMatch(regex, "Gian "); // Fails: Cannot end with a space
534+
}
535+
536+
@Test
537+
@DisplayName("Should validate strings containing Unicode symbols like currencies or mathematical operators")
538+
void testUnicodeSymbols() {
539+
// Goal: Match a generic price tag, mathematical notation, or symbol-based input.
540+
// Format: [Symbol] [Optional Space] [Digits]
541+
542+
// 1. The symbol (Currency, Math, Dingbats, etc. -> \p{S})
543+
CharacterConnector<Fragment> symbol = exactly(1).symbolsUnicode();
544+
545+
// 2. An optional space separator
546+
Connector<Fragment> optionalSpace = optional().whitespace();
547+
548+
// 3. The numeric value
549+
Connector<Fragment> amount = oneOrMore().digits();
550+
551+
String regex = Sift.fromStart()
552+
.of(symbol)
553+
.followedBy(optionalSpace)
554+
.followedBy(amount)
555+
.andNothingElse()
556+
.shake();
557+
558+
// Valid symbol inputs
559+
assertMatches(regex, "€ 100"); // Euro symbol (Currency)
560+
assertMatches(regex, "¥5000"); // Yen symbol (No space)
561+
assertMatches(regex, "$ 1"); // Dollar symbol (Currency)
562+
assertMatches(regex, "∑ 42"); // Sum symbol (Mathematical)
563+
assertMatches(regex, "♥ 2"); // Heart symbol (Miscellaneous/Dingbat)
564+
565+
// Invalid cases where standard letters are used instead of Unicode symbols
566+
assertDoesNotMatch(regex, "EUR 100"); // 'E' is a letter, not a symbol (\p{S})
567+
assertDoesNotMatch(regex, "USD 50"); // Fails on letters
568+
assertDoesNotMatch(regex, "100"); // Missing the required symbol at the start
569+
}
468570
}

0 commit comments

Comments
 (0)