[GR-80056] TRegex: fix bugs found by fuzzing.

PullRequest: graal/25556

Author: djoooooe

regex/src/com.oracle.truffle.regex.test/src/com/oracle/truffle/regex/tregex/test/generated/OracleDBGeneratedTests.java

@@ -1727,6 +1727,9 @@ public class OracleDBGeneratedTests {
         testCase("w[v-w]\\W", "", UTF_8, match("w\ud839\udfefwww\ud839\udfefw\ud839\udfefw", 5, 6, 12)),
         testCase("(a{1100,1100})\\1", "i", UTF_8, match("a".repeat(2400), 0, 0, 2200, 0, 1100)),
         testCase("[a]\\S{213,213}bcdz", "", UTF_8, noMatch("a".repeat(215) + ("bcxd" + "a".repeat(213)).repeat(3), 0)),
+        testCase("z()|(d)[[=\ud97a\udcb9=]\ud9ba\udcb9]", "m", UTF_16BE, match("zz", 0, 0, 1, 1, 1, -1, -1)),
+        testCase("(|\udbda\udcf5)[[.\uda83\udd45.][.\uda43\udd45.]]", "", UTF_16BE, match("\udbda\udcf5\uda83\udd45", 0, 0, 4, 0, 2)),
+        testCase("[[=\u01c4=]]", "", UTF_8, match("\u01c6", 0, 0, 2)),
 
         /* GENERATED CODE END - KEEP THIS MARKER FOR AUTOMATIC UPDATES */
         // Checkstyle: resume

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/MultiCharacterCaseFolding.java

@@ -258,18 +258,17 @@ public static void caseClosure(CaseFoldData.CaseFoldAlgorithm algorithm, CodePoi
                     CodePointSet allowedCodePoints, boolean transitiveEquivalence) {
         tmp.clear();
         caseFoldCharClass(algorithm, charClass, (from, to) -> {
-            if (transitiveEquivalence || hasNoCaseFolding(algorithm, to[0])) {
-                if (to.length == 1) {
-                    // Add the case-folded version to the character class...
-                    if (filter.test(from, to[0])) {
-                        tmp.addCodePoint(to[0]);
-                    }
+            if (to.length == 1 && (transitiveEquivalence || hasNoCaseFolding(algorithm, to[0]))) {
+                // Add the case-folded version to the character class.
+                if (filter.test(from, to[0])) {
+                    tmp.addCodePoint(to[0]);
                 }
-                // ... and also any characters which case-fold to the same.
-                for (int unfolding : CaseUnfoldingTrie.findSingleCharUnfoldings(algorithm, to)) {
-                    if (unfolding != from && filter.test(from, unfolding)) {
-                        tmp.addCodePoint(unfolding);
-                    }
+            }
+            // Add any characters which case-fold to the same, even if the common case-folded value
+            // itself has another case-folding.
+            for (int unfolding : CaseUnfoldingTrie.findSingleCharUnfoldings(algorithm, to)) {
+                if (unfolding != from && filter.test(from, unfolding)) {
+                    tmp.addCodePoint(unfolding);
                 }
             }
         });

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/RegexASTPostProcessor.java

@@ -127,8 +127,7 @@ private void checkInnerLiteral() {
 
     private boolean isLiteralChar(Term t) {
         return t.isCharacterClass() &&
-                        (t.asCharacterClass().getCharSet().matchesSingleChar() || t.asCharacterClass().getCharSet().matches2CharsWith1BitDifference()) &&
-                        ast.getEncoding().isFixedCodePointWidth(t.asCharacterClass().getCharSet()) &&
+                        ast.getEncoding().canBeMatchedWithMask(t.asCharacterClass().getCharSet()) &&
                         !(ast.getEncoding().isUTF16() && t.asCharacterClass().getCharSet().intersects(Constants.SURROGATES));
     }
 

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/ast/CalcASTPropsVisitor.java

@@ -579,7 +579,7 @@ static void setFlagsSubtreeRootNode(RegexASTSubtreeRootNode subtreeRootNode, int
     protected void visit(CharacterClass characterClass) {
         if (isForward()) {
             if (!characterClass.getCharSet().matchesSingleChar()) {
-                if (!characterClass.getCharSet().matches2CharsWith1BitDifference()) {
+                if (!ast.getEncoding().canBeMatchedWithMask(characterClass.getCharSet())) {
                     ast.getProperties().unsetCharClassesCanBeMatchedWithMask();
                 }
                 if (!ast.getEncoding().isFixedCodePointWidth(characterClass.getCharSet())) {

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/ast/CharacterClass.java

@@ -183,7 +183,7 @@ public void extractSingleChar(AbstractStringBuffer literal, AbstractStringBuffer
                 mask.append(0);
             }
         } else {
-            assert charSet.matches2CharsWith1BitDifference();
+            assert mask.getEncoding().canBeMatchedWithMask(charSet);
             int c1 = charSet.getMin();
             int c2 = charSet.getMax();
             literal.appendOR(c1, c2);

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/ast/RegexAST.java

@@ -707,10 +707,9 @@ public InnerLiteral extractInnerLiteral() {
         boolean hasMask = false;
         for (int i = literalStart; i < literalEnd; i++) {
             CharacterClass cc = root.getFirstAlternative().getTerms().get(i).asCharacterClass();
-            assert cc.getCharSet().matchesSingleChar() || cc.getCharSet().matches2CharsWith1BitDifference();
-            assert getEncoding().isFixedCodePointWidth(cc.getCharSet());
+            assert getEncoding().canBeMatchedWithMask(cc.getCharSet());
             cc.extractSingleChar(literal, mask);
-            hasMask |= cc.getCharSet().matches2CharsWith1BitDifference();
+            hasMask |= !cc.getCharSet().matchesSingleChar();
         }
         int maxPrefixSize = root.getFirstAlternative().get(literalStart).getMaxPath() - 1;
         for (int i = 0; i < literalStart; i++) {

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/string/Encoding.java

@@ -202,6 +202,26 @@ public boolean isFixedCodePointWidth(CodePointSet set) {
         }
     }
 
+    /**
+     * Returns {@code true} if the given code point set can be represented as a literal and a bit
+     * mask without matching any additional code points.
+     */
+    public boolean canBeMatchedWithMask(CodePointSet set) {
+        if (set.matchesSingleChar()) {
+            return true;
+        }
+        if (!set.matches2CharsWith1BitDifference() || !isFixedCodePointWidth(set)) {
+            return false;
+        }
+        if (isUTF16() && set.getMin() > Character.MAX_VALUE) {
+            int c1 = set.getMin();
+            int c2 = set.getMax();
+            return Integer.bitCount(Character.highSurrogate(c1) ^ Character.highSurrogate(c2)) +
+                            Integer.bitCount(Character.lowSurrogate(c1) ^ Character.lowSurrogate(c2)) == 1;
+        }
+        return true;
+    }
+
     public boolean isUnicode() {
         return switch (this) {
             case LATIN_1, ASCII, BYTES -> false;