From ba066abe2a3106edda0073e5eb4eabeb18ce0640 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Mon, 10 Aug 2026 13:47:22 +0200 Subject: [PATCH] fix: complete ICU-backed regex folds Use the full ICU casemapped candidate set, supplement simple folds missing from Java Pattern's Unicode tables, and include sibling code points sharing the same full fold. Generated with [Codex](https://developers.openai.com/codex/) Co-Authored-By: Codex --- .../runtime/regex/MultiCharFoldMapper.java | 63 ++++++++++++++++++- .../runtime/regex/RegexPreprocessor.java | 19 ++++-- .../regex/MultiCharFoldMapperTest.java | 30 +++++++++ .../resources/unit/regex_icu_full_casefold.t | 20 ++++++ 4 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java create mode 100644 src/test/resources/unit/regex_icu_full_casefold.t diff --git a/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java b/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java index 244f28e90..3eda2ad5a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java +++ b/src/main/java/org/perlonjava/runtime/regex/MultiCharFoldMapper.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -27,21 +28,45 @@ public class MultiCharFoldMapper { // Format: fold string → characters private static final Map> REVERSE_FOLDS = new HashMap<>(); private static final Set FOLD_COMPONENTS = new HashSet<>(); + private static final Map> SIMPLE_FOLD_CLASSES = new HashMap<>(); static { // ICU4J is already the runtime's Unicode source of truth. Derive all // full folds instead of maintaining a partial hand-written table. + // CHANGES_WHEN_CASEMAPPED is intentionally used as a superset: + // CHANGES_WHEN_CASEFOLDED excludes lowercase characters such as + // U+01F0 and U+0390 even though their full fold has multiple code + // points. UnicodeSet foldCandidates = new UnicodeSet() - .applyIntPropertyValue(UProperty.CHANGES_WHEN_CASEFOLDED, 1); + .applyIntPropertyValue(UProperty.CHANGES_WHEN_CASEMAPPED, 1); + Map> simpleFoldClasses = new HashMap<>(); for (String original : foldCandidates) { if (original.codePointCount(0, original.length()) != 1) continue; int codePoint = original.codePointAt(0); String fold = UCharacter.foldCase(original, true); if (fold.codePointCount(0, fold.length()) > 1) { MULTI_CHAR_FOLDS.put(codePoint, fold); + } else { + int foldedCodePoint = fold.codePointAt(0); + LinkedHashSet foldClass = simpleFoldClasses.computeIfAbsent( + foldedCodePoint, ignored -> new LinkedHashSet<>()); + foldClass.add(foldedCodePoint); + foldClass.add(codePoint); } } + for (LinkedHashSet foldClass : simpleFoldClasses.values()) { + if (foldClass.size() < 2) continue; + List variants = List.copyOf(foldClass); + String representative = new String(Character.toChars(variants.get(0))); + Pattern javaFoldPattern = Pattern.compile(Pattern.quote(representative), + Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE); + boolean javaSupportsFoldClass = variants.stream().allMatch(codePoint -> + javaFoldPattern.matcher(new String(Character.toChars(codePoint))).matches()); + if (javaSupportsFoldClass) continue; + for (int codePoint : variants) SIMPLE_FOLD_CLASSES.put(codePoint, variants); + } + // Build reverse map (lowercase versions only for simpler matching) for (Map.Entry entry : MULTI_CHAR_FOLDS.entrySet()) { String fold = entry.getValue(); @@ -87,10 +112,16 @@ public static String expandToAlternation(int codePoint) { StringBuilder sb = new StringBuilder("(?:"); String original = new String(Character.toChars(codePoint)); sb.append(Pattern.quote(original)); - sb.append("|"); + + // Include sibling code points with the same full fold. For example, + // both U+00DF and U+1E9E fold to "ss". + for (int reverseFold : REVERSE_FOLDS.getOrDefault(fold, List.of())) { + if (reverseFold == codePoint) continue; + sb.append('|').append(Pattern.quote(new String(Character.toChars(reverseFold)))); + } // Add the basic fold - sb.append(Pattern.quote(fold)); + sb.append('|').append(Pattern.quote(fold)); // Add case variations of the fold (if it's ASCII) if (fold.chars().allMatch(c -> c >= 'a' && c <= 'z')) { @@ -136,6 +167,32 @@ public static List getReverseFolds(String str) { return folds == null ? List.of() : folds; } + /** Whether ICU knows a non-trivial single-code-point fold for this literal. */ + public static boolean hasSimpleFold(int codePoint) { + return SIMPLE_FOLD_CLASSES.containsKey(codePoint); + } + + /** + * Expand a single-code-point fold class into a quoted alternation. This + * supplements Java Pattern for characters newer than the JDK's Unicode + * tables while remaining harmless for fold classes Java already knows. + */ + public static String expandSimpleFoldToAlternation(int codePoint) { + List variants = SIMPLE_FOLD_CLASSES.get(codePoint); + if (variants == null) return null; + StringBuilder result = new StringBuilder("(?:"); + for (int i = 0; i < variants.size(); i++) { + if (i > 0) result.append('|'); + result.append(Pattern.quote(new String(Character.toChars(variants.get(i))))); + } + return result.append(')').toString(); + } + + /** Get all members of a simple fold class, or an empty list. */ + public static List getSimpleFoldVariants(int codePoint) { + return SIMPLE_FOLD_CLASSES.getOrDefault(codePoint, List.of()); + } + /** Whether a literal code point can participate in a reverse multi-char fold. */ public static boolean isFoldComponent(int codePoint) { if (FOLD_COMPONENTS.contains(codePoint)) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index eb5d03f65..5c228805b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -995,8 +995,11 @@ private static String expandMultiCharFolds(String pattern, RegexFlags regexFlags } if (!foundReverseFold) { - String specialExpansion = regexFlags.isAsciiStrict() - ? null : expandSpecialSingleCharFold(codePoint); + String specialExpansion = regexFlags.isAsciiStrict() ? null + : MultiCharFoldMapper.expandSimpleFoldToAlternation(codePoint); + if (specialExpansion == null && !regexFlags.isAsciiStrict()) { + specialExpansion = expandSpecialSingleCharFold(codePoint); + } if (specialExpansion != null) { result.append(specialExpansion); } else { @@ -1016,6 +1019,7 @@ private static String expandMultiCharFoldClass(String charClass, RegexFlags rege if (regexFlags.isAsciiStrict() || charClass.length() < 3 || charClass.charAt(1) == '^') return null; LinkedHashSet folds = new LinkedHashSet<>(); + LinkedHashSet simpleVariants = new LinkedHashSet<>(); boolean escaped = false; for (int i = 1; i < charClass.length() - 1; ) { int codePoint = charClass.codePointAt(i); @@ -1032,11 +1036,17 @@ private static String expandMultiCharFoldClass(String charClass, RegexFlags rege if (codePoint == '-' || codePoint == '[' || codePoint == ']') return null; String fold = MultiCharFoldMapper.getMultiCharFold(codePoint); if (fold != null) folds.add(Pattern.quote(fold)); + simpleVariants.addAll(MultiCharFoldMapper.getSimpleFoldVariants(codePoint)); i += Character.charCount(codePoint); } - if (folds.isEmpty()) return null; + if (folds.isEmpty() && simpleVariants.isEmpty()) return null; + + StringBuilder expandedClass = new StringBuilder(charClass.substring(0, charClass.length() - 1)); + simpleVariants.forEach(expandedClass::appendCodePoint); + expandedClass.append(']'); + if (folds.isEmpty()) return expandedClass.toString(); - StringBuilder expansion = new StringBuilder("(?:").append(charClass); + StringBuilder expansion = new StringBuilder("(?:").append(expandedClass); for (String fold : folds) expansion.append('|').append(fold); return expansion.append(')').toString(); } @@ -1058,6 +1068,7 @@ private static String materializeFoldableHexEscapes(String pattern) { int codePoint = Integer.parseInt(pattern.substring(i + 3, close), 16); if (Character.isValidCodePoint(codePoint) && (MultiCharFoldMapper.hasMultiCharFold(codePoint) + || MultiCharFoldMapper.hasSimpleFold(codePoint) || MultiCharFoldMapper.isFoldComponent(codePoint))) { result.appendCodePoint(codePoint); i = close + 1; diff --git a/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java b/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java new file mode 100644 index 000000000..393132fe2 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/MultiCharFoldMapperTest.java @@ -0,0 +1,30 @@ +package org.perlonjava.runtime.regex; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("unit") +class MultiCharFoldMapperTest { + @Test + void includesLowercaseCharactersWithFullFolds() { + assertEquals("j\u030C", MultiCharFoldMapper.getMultiCharFold(0x01F0)); + assertEquals("\u03B9\u0308\u0301", MultiCharFoldMapper.getMultiCharFold(0x0390)); + } + + @Test + void includesEveryCodePointSharingAFullFold() { + assertTrue(MultiCharFoldMapper.getReverseFolds("ss").containsAll(List.of(0x00DF, 0x1E9E))); + assertTrue(MultiCharFoldMapper.expandToAlternation(0x00DF).contains("\u1E9E")); + } + + @Test + void supplementsSimpleFoldsMissingFromJavaPattern() { + assertTrue(MultiCharFoldMapper.getSimpleFoldVariants(0xA7CE).contains(0xA7CF)); + assertTrue(MultiCharFoldMapper.getSimpleFoldVariants(0x16EA0).contains(0x16EBB)); + } +} diff --git a/src/test/resources/unit/regex_icu_full_casefold.t b/src/test/resources/unit/regex_icu_full_casefold.t new file mode 100644 index 000000000..6e2d3c19f --- /dev/null +++ b/src/test/resources/unit/regex_icu_full_casefold.t @@ -0,0 +1,20 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use utf8; +use Test::More tests => 6; + +ok("\x{01F0}" =~ /^\x{006A}\x{030C}$/iu, + 'lowercase j with caron expands to its full fold'); +ok("\x{006A}\x{030C}" =~ /^\x{01F0}$/iu, + 'j plus combining caron reverse-folds to one code point'); + +ok("\x{0390}" =~ /^\x{03B9}\x{0308}\x{0301}$/iu, + 'Greek lowercase character expands to its three-code-point fold'); +ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iu, + 'Greek three-code-point sequence reverse-folds'); + +ok("\x{1E9E}" =~ /^\x{00DF}$/iu, + 'capital sharp s matches its sibling full-fold source'); +ok("\x{00DF}" =~ /^\x{1E9E}$/iu, + 'sibling full-fold source matching is symmetric');