diff --git a/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java b/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java index 244f28e90..3eda2ad5a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java +++ b/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -27,21 +28,45 @@ public class MultiCharFoldMapper { // Format: fold string → characters private static final Map> REVERSE_FOLDS = new HashMap<>(); private static final Set FOLD_COMPONENTS = new HashSet<>(); + private static final Map> SIMPLE_FOLD_CLASSES = new HashMap<>(); static { // ICU4J is already the runtime's Unicode source of truth. Derive all // full folds instead of maintaining a partial hand-written table. + // CHANGES_WHEN_CASEMAPPED is intentionally used as a superset: + // CHANGES_WHEN_CASEFOLDED excludes lowercase characters such as + // U+01F0 and U+0390 even though their full fold has multiple code + // points. UnicodeSet foldCandidates = new UnicodeSet() - .applyIntPropertyValue(UProperty.CHANGES_WHEN_CASEFOLDED, 1); + .applyIntPropertyValue(UProperty.CHANGES_WHEN_CASEMAPPED, 1); + Map> simpleFoldClasses = new HashMap<>(); for (String original : foldCandidates) { if (original.codePointCount(0, original.length()) != 1) continue; int codePoint = original.codePointAt(0); String fold = UCharacter.foldCase(original, true); if (fold.codePointCount(0, fold.length()) > 1) { MULTI_CHAR_FOLDS.put(codePoint, fold); + } else { + int foldedCodePoint = fold.codePointAt(0); + LinkedHashSet foldClass = simpleFoldClasses.computeIfAbsent( + foldedCodePoint, ignored -> new LinkedHashSet<>()); + foldClass.add(foldedCodePoint); + foldClass.add(codePoint); } } + for (LinkedHashSet foldClass : simpleFoldClasses.values()) { + if (foldClass.size() < 2) continue; + List variants = List.copyOf(foldClass); + String representative = new String(Character.toChars(variants.get(0))); + Pattern javaFoldPattern = Pattern.compile(Pattern.quote(representative), + Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE); + boolean javaSupportsFoldClass = variants.stream().allMatch(codePoint -> + javaFoldPattern.matcher(new String(Character.toChars(codePoint))).matches()); + if (javaSupportsFoldClass) continue; + for (int codePoint : variants) SIMPLE_FOLD_CLASSES.put(codePoint, variants); + } + // Build reverse map (lowercase versions only for simpler matching) for (Map.Entry entry : MULTI_CHAR_FOLDS.entrySet()) { String fold = entry.getValue(); @@ -87,10 +112,16 @@ public static String expandToAlternation(int codePoint) { StringBuilder sb = new StringBuilder("(?:"); String original = new String(Character.toChars(codePoint)); sb.append(Pattern.quote(original)); - sb.append("|"); + + // Include sibling code points with the same full fold. For example, + // both U+00DF and U+1E9E fold to "ss". + for (int reverseFold : REVERSE_FOLDS.getOrDefault(fold, List.of())) { + if (reverseFold == codePoint) continue; + sb.append('|').append(Pattern.quote(new String(Character.toChars(reverseFold)))); + } // Add the basic fold - sb.append(Pattern.quote(fold)); + sb.append('|').append(Pattern.quote(fold)); // Add case variations of the fold (if it's ASCII) if (fold.chars().allMatch(c -> c >= 'a' && c <= 'z')) { @@ -136,6 +167,32 @@ public static List getReverseFolds(String str) { return folds == null ? List.of() : folds; } + /** Whether ICU knows a non-trivial single-code-point fold for this literal. */ + public static boolean hasSimpleFold(int codePoint) { + return SIMPLE_FOLD_CLASSES.containsKey(codePoint); + } + + /** + * Expand a single-code-point fold class into a quoted alternation. This + * supplements Java Pattern for characters newer than the JDK's Unicode + * tables while remaining harmless for fold classes Java already knows. + */ + public static String expandSimpleFoldToAlternation(int codePoint) { + List variants = SIMPLE_FOLD_CLASSES.get(codePoint); + if (variants == null) return null; + StringBuilder result = new StringBuilder("(?:"); + for (int i = 0; i < variants.size(); i++) { + if (i > 0) result.append('|'); + result.append(Pattern.quote(new String(Character.toChars(variants.get(i))))); + } + return result.append(')').toString(); + } + + /** Get all members of a simple fold class, or an empty list. */ + public static List getSimpleFoldVariants(int codePoint) { + return SIMPLE_FOLD_CLASSES.getOrDefault(codePoint, List.of()); + } + /** Whether a literal code point can participate in a reverse multi-char fold. */ public static boolean isFoldComponent(int codePoint) { if (FOLD_COMPONENTS.contains(codePoint)) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index eb5d03f65..5c228805b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -995,8 +995,11 @@ private static String expandMultiCharFolds(String pattern, RegexFlags regexFlags } if (!foundReverseFold) { - String specialExpansion = regexFlags.isAsciiStrict() - ? null : expandSpecialSingleCharFold(codePoint); + String specialExpansion = regexFlags.isAsciiStrict() ? null + : MultiCharFoldMapper.expandSimpleFoldToAlternation(codePoint); + if (specialExpansion == null && !regexFlags.isAsciiStrict()) { + specialExpansion = expandSpecialSingleCharFold(codePoint); + } if (specialExpansion != null) { result.append(specialExpansion); } else { @@ -1016,6 +1019,7 @@ private static String expandMultiCharFoldClass(String charClass, RegexFlags rege if (regexFlags.isAsciiStrict() || charClass.length() < 3 || charClass.charAt(1) == '^') return null; LinkedHashSet folds = new LinkedHashSet<>(); + LinkedHashSet simpleVariants = new LinkedHashSet<>(); boolean escaped = false; for (int i = 1; i < charClass.length() - 1; ) { int codePoint = charClass.codePointAt(i); @@ -1032,11 +1036,17 @@ private static String expandMultiCharFoldClass(String charClass, RegexFlags rege if (codePoint == '-' || codePoint == '[' || codePoint == ']') return null; String fold = MultiCharFoldMapper.getMultiCharFold(codePoint); if (fold != null) folds.add(Pattern.quote(fold)); + simpleVariants.addAll(MultiCharFoldMapper.getSimpleFoldVariants(codePoint)); i += Character.charCount(codePoint); } - if (folds.isEmpty()) return null; + if (folds.isEmpty() && simpleVariants.isEmpty()) return null; + + StringBuilder expandedClass = new StringBuilder(charClass.substring(0, charClass.length() - 1)); + simpleVariants.forEach(expandedClass::appendCodePoint); + expandedClass.append(']'); + if (folds.isEmpty()) return expandedClass.toString(); - StringBuilder expansion = new StringBuilder("(?:").append(charClass); + StringBuilder expansion = new StringBuilder("(?:").append(expandedClass); for (String fold : folds) expansion.append('|').append(fold); return expansion.append(')').toString(); } @@ -1058,6 +1068,7 @@ private static String materializeFoldableHexEscapes(String pattern) { int codePoint = Integer.parseInt(pattern.substring(i + 3, close), 16); if (Character.isValidCodePoint(codePoint) && (MultiCharFoldMapper.hasMultiCharFold(codePoint) + || MultiCharFoldMapper.hasSimpleFold(codePoint) || MultiCharFoldMapper.isFoldComponent(codePoint))) { result.appendCodePoint(codePoint); i = close + 1; diff --git a/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java b/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java new file mode 100644 index 000000000..393132fe2 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java @@ -0,0 +1,30 @@ +package org.perlonjava.runtime.regex; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("unit") +class MultiCharFoldMapperTest { + @Test + void includesLowercaseCharactersWithFullFolds() { + assertEquals("j\u030C", MultiCharFoldMapper.getMultiCharFold(0x01F0)); + assertEquals("\u03B9\u0308\u0301", MultiCharFoldMapper.getMultiCharFold(0x0390)); + } + + @Test + void includesEveryCodePointSharingAFullFold() { + assertTrue(MultiCharFoldMapper.getReverseFolds("ss").containsAll(List.of(0x00DF, 0x1E9E))); + assertTrue(MultiCharFoldMapper.expandToAlternation(0x00DF).contains("\u1E9E")); + } + + @Test + void supplementsSimpleFoldsMissingFromJavaPattern() { + assertTrue(MultiCharFoldMapper.getSimpleFoldVariants(0xA7CE).contains(0xA7CF)); + assertTrue(MultiCharFoldMapper.getSimpleFoldVariants(0x16EA0).contains(0x16EBB)); + } +} diff --git a/src/test/resources/unit/regex_icu_full_casefold.t b/src/test/resources/unit/regex_icu_full_casefold.t new file mode 100644 index 000000000..6e2d3c19f --- /dev/null +++ b/src/test/resources/unit/regex_icu_full_casefold.t @@ -0,0 +1,20 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use utf8; +use Test::More tests => 6; + +ok("\x{01F0}" =~ /^\x{006A}\x{030C}$/iu, + 'lowercase j with caron expands to its full fold'); +ok("\x{006A}\x{030C}" =~ /^\x{01F0}$/iu, + 'j plus combining caron reverse-folds to one code point'); + +ok("\x{0390}" =~ /^\x{03B9}\x{0308}\x{0301}$/iu, + 'Greek lowercase character expands to its three-code-point fold'); +ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iu, + 'Greek three-code-point sequence reverse-folds'); + +ok("\x{1E9E}" =~ /^\x{00DF}$/iu, + 'capital sharp s matches its sibling full-fold source'); +ok("\x{00DF}" =~ /^\x{1E9E}$/iu, + 'sibling full-fold source matching is symmetric');