Fix a crash when a pylint must display unicode raising a UnicodeEncodeError

doc/whatsnew/fragments/8736.bugfix

@@ -0,0 +1,5 @@
+When displaying unicode with surrogates (or other potential ``UnicodeEncodeError``),
+pylint will now display a '?' character (using ``encode(encoding="utf-8", errors="replace")``)
+instead of crashing. The functional tests classes are also updated to handle this case.
+
+Closes #8736.

doc/whatsnew/fragments/8736.feature

@@ -0,0 +1,5 @@
+``comparison-of-constants`` now use the unicode from the ast instead of reformatting from
+ the node's values preventing some bad formatting due to ``utf-8`` limitation. The message now use
+ ``"`` instead of ``'`` to better work with what the python ast returns.
+
+Refs #8736

pylint/checkers/base/comparison_checker.py

@@ -60,7 +60,7 @@ class ComparisonChecker(_BasicChecker):
             "Used when something is compared against itself.",
         ),
         "R0133": (
-            "Comparison between constants: '%s %s %s' has a constant value",
+            'Comparison between constants: "%s %s %s" has a constant value',
             "comparison-of-constants",
             "When two literals are compared with each other the result is a constant. "
             "Using the constant directly is both easier to read and more performant. "
@@ -257,7 +257,7 @@ def _check_constants_comparison(self, node: nodes.Compare) -> None:
         self.add_message(
             "comparison-of-constants",
             node=node,
-            args=(left_operand.value, operator, right_operand.value),
+            args=(left_operand.as_string(), operator, right_operand.as_string()),
             confidence=HIGH,
         )
 

pylint/reporters/base_reporter.py

@@ -42,7 +42,14 @@ def handle_message(self, msg: Message) -> None:
 
     def writeln(self, string: str = "") -> None:
         """Write a line in the output buffer."""
-        print(string, file=self.out)
+        try:
+            print(string, file=self.out)
+        except UnicodeEncodeError:
+            print(self.reencode_output_after_unicode_error(string), file=self.out)
+
+    @staticmethod
+    def reencode_output_after_unicode_error(string: str) -> str:
+        return string.encode(encoding="utf-8", errors="replace").decode("utf8")
 
     def display_reports(self, layout: Section) -> None:
         """Display results encapsulated in the layout tree."""

pylint/testutils/functional/lint_module_output_update.py

@@ -40,4 +40,4 @@ def _check_output_text(
         with open(self._test_file.expected_output, "w", encoding="utf-8") as f:
             writer = csv.writer(f, dialect="test")
             for line in actual_output:
-                writer.writerow(line.to_csv())
+                self.safe_write_output_line(writer, line)

pylint/testutils/lint_module_test.py

@@ -11,7 +11,7 @@
 from collections import Counter
 from io import StringIO
 from pathlib import Path
-from typing import TextIO
+from typing import TYPE_CHECKING, TextIO
 
 import pytest
 from _pytest.config import Config
@@ -20,6 +20,7 @@
 from pylint.config.config_initialization import _config_initialization
 from pylint.lint import PyLinter
 from pylint.message.message import Message
+from pylint.reporters import BaseReporter
 from pylint.testutils.constants import _EXPECTED_RE, _OPERATORS, UPDATE_OPTION
 
 # need to import from functional.test_file to avoid cyclic import
@@ -31,6 +32,8 @@
 from pylint.testutils.output_line import OutputLine
 from pylint.testutils.reporter_for_tests import FunctionalTestReporter
 
+if TYPE_CHECKING:
+    import _csv
 MessageCounter = Counter[tuple[int, str]]
 
 PYLINTRC = Path(__file__).parent / "testing_pylintrc"
@@ -309,10 +312,22 @@ def error_msg_for_unequal_output(
             expected_csv = StringIO()
             writer = csv.writer(expected_csv, dialect="test")
             for line in sorted(received_lines, key=sort_by_line_number):
-                writer.writerow(line.to_csv())
+                self.safe_write_output_line(writer, line)
             error_msg += expected_csv.getvalue()
         return error_msg
 
+    def safe_write_output_line(self, writer: _csv._writer, line: OutputLine) -> None:
+        """Write an OutputLine to the CSV writer, handling UnicodeEncodeError."""
+        try:
+            writer.writerow(line.to_csv())
+        except UnicodeEncodeError:
+            writer.writerow(
+                [
+                    BaseReporter.reencode_output_after_unicode_error(s)
+                    for s in line.to_csv()
+                ]
+            )
+
     def _check_output_text(
         self,
         _: MessageCounter,

tests/functional/c/comparison_of_constants.txt

@@ -1,4 +1,4 @@
-comparison-of-constants:3:6:3:12::"Comparison between constants: '2 == 2' has a constant value":HIGH
-comparison-of-constants:6:6:6:11::"Comparison between constants: '2 > 2' has a constant value":HIGH
-comparison-of-constants:16:3:16:15::"Comparison between constants: 'True == True' has a constant value":HIGH
+comparison-of-constants:3:6:3:12::"Comparison between constants: ""2 == 2"" has a constant value":HIGH
+comparison-of-constants:6:6:6:11::"Comparison between constants: ""2 > 2"" has a constant value":HIGH
+comparison-of-constants:16:3:16:15::"Comparison between constants: ""True == True"" has a constant value":HIGH
 singleton-comparison:16:3:16:15::Comparison 'True == True' should be 'True is True' if checking for the singleton value True, or 'True' if testing for truthiness:UNDEFINED

tests/functional/ext/magic_value_comparison/magic_value_comparison.txt

@@ -1,7 +1,7 @@
 magic-value-comparison:16:3:16:10::Consider using a named constant or an enum instead of '5'.:HIGH
 magic-value-comparison:19:3:19:17::Consider using a named constant or an enum instead of '10'.:HIGH
 magic-value-comparison:22:9:22:18::Consider using a named constant or an enum instead of '100'.:HIGH
-comparison-of-constants:24:17:24:22::"Comparison between constants: '5 > 7' has a constant value":HIGH
+comparison-of-constants:24:17:24:22::"Comparison between constants: ""5 > 7"" has a constant value":HIGH
 singleton-comparison:29:17:29:28::Comparison 'var == True' should be 'var is True' if checking for the singleton value True, or 'bool(var)' if testing for truthiness:UNDEFINED
 singleton-comparison:30:17:30:29::Comparison 'var == False' should be 'var is False' if checking for the singleton value False, or 'not var' if testing for falsiness:UNDEFINED
 singleton-comparison:31:17:31:28::Comparison 'var == None' should be 'var is None':UNDEFINED

tests/functional/l/literal_comparison.txt

@@ -1,8 +1,8 @@
-comparison-of-constants:4:3:4:9::"Comparison between constants: '2 is 2' has a constant value":HIGH
+comparison-of-constants:4:3:4:9::"Comparison between constants: ""2 is 2"" has a constant value":HIGH
 literal-comparison:4:3:4:9::In '2 is 2', use '==' when comparing constant literals not 'is' ('2 == 2'):HIGH
-comparison-of-constants:7:3:7:14::"Comparison between constants: 'a is b'a'' has a constant value":HIGH
+comparison-of-constants:7:3:7:14::"Comparison between constants: ""'a' is b'a'"" has a constant value":HIGH
 literal-comparison:7:3:7:14::In ''a' is b'a'', use '==' when comparing constant literals not 'is' (''a' == b'a''):HIGH
-comparison-of-constants:10:3:10:13::"Comparison between constants: '2.0 is 3.0' has a constant value":HIGH
+comparison-of-constants:10:3:10:13::"Comparison between constants: ""2.0 is 3.0"" has a constant value":HIGH
 literal-comparison:10:3:10:13::In '2.0 is 3.0', use '==' when comparing constant literals not 'is' ('2.0 == 3.0'):HIGH
 literal-comparison:16:3:16:19::"In '() is {1: 2, 2: 3}', use '==' when comparing constant literals not 'is' ('() == {1: 2, 2: 3}')":HIGH
 literal-comparison:19:3:19:18::In '[] is [4, 5, 6]', use '==' when comparing constant literals not 'is' ('[] == [4, 5, 6]'):HIGH

tests/functional/l/logical_tautology.txt

@@ -2,17 +2,17 @@ comparison-with-itself:6:7:6:17:foo:Redundant comparison - arg == arg:UNDEFINED
 comparison-with-itself:8:9:8:19:foo:Redundant comparison - arg != arg:UNDEFINED
 comparison-with-itself:10:9:10:18:foo:Redundant comparison - arg > arg:UNDEFINED
 comparison-with-itself:12:9:12:19:foo:Redundant comparison - arg <= arg:UNDEFINED
-comparison-of-constants:14:9:14:21:foo:"Comparison between constants: 'None == None' has a constant value":HIGH
+comparison-of-constants:14:9:14:21:foo:"Comparison between constants: ""None == None"" has a constant value":HIGH
 comparison-with-itself:14:9:14:21:foo:Redundant comparison - None == None:UNDEFINED
-comparison-of-constants:16:9:16:19:foo:"Comparison between constants: '786 == 786' has a constant value":HIGH
+comparison-of-constants:16:9:16:19:foo:"Comparison between constants: ""786 == 786"" has a constant value":HIGH
 comparison-with-itself:16:9:16:19:foo:Redundant comparison - 786 == 786:UNDEFINED
-comparison-of-constants:18:9:18:19:foo:"Comparison between constants: '786 is 786' has a constant value":HIGH
+comparison-of-constants:18:9:18:19:foo:"Comparison between constants: ""786 is 786"" has a constant value":HIGH
 comparison-with-itself:18:9:18:19:foo:Redundant comparison - 786 is 786:UNDEFINED
-comparison-of-constants:20:9:20:23:foo:"Comparison between constants: '786 is not 786' has a constant value":HIGH
+comparison-of-constants:20:9:20:23:foo:"Comparison between constants: ""786 is not 786"" has a constant value":HIGH
 comparison-with-itself:20:9:20:23:foo:Redundant comparison - 786 is not 786:UNDEFINED
 comparison-with-itself:22:9:22:19:foo:Redundant comparison - arg is arg:UNDEFINED
 comparison-with-itself:24:9:24:23:foo:Redundant comparison - arg is not arg:UNDEFINED
-comparison-of-constants:26:9:26:21:foo:"Comparison between constants: 'True is True' has a constant value":HIGH
+comparison-of-constants:26:9:26:21:foo:"Comparison between constants: ""True is True"" has a constant value":HIGH
 comparison-with-itself:26:9:26:21:foo:Redundant comparison - True is True:UNDEFINED
-comparison-of-constants:28:9:28:19:foo:"Comparison between constants: '666 == 786' has a constant value":HIGH
+comparison-of-constants:28:9:28:19:foo:"Comparison between constants: ""666 == 786"" has a constant value":HIGH
 comparison-with-itself:36:18:36:28:bar:Redundant comparison - arg != arg:UNDEFINED

tests/functional/r/regression_02/regression_8736.py

@@ -0,0 +1,3 @@
+"""This does not crash in the functional tests, but it did when called directly."""
+
+assert "\U00010000" == "\ud800\udc00"  # [comparison-of-constants]

tests/functional/r/regression_02/regression_8736.txt

@@ -0,0 +1 @@
+comparison-of-constants:3:7:3:37::"Comparison between constants: ""'𐀀' == '\ud800\udc00'"" has a constant value":HIGH

tests/functional/u/use/use_implicit_booleaness_not_len.txt

@@ -2,7 +2,7 @@ use-implicit-booleaness-not-len:4:3:4:14::Do not use `len(SEQUENCE)` without com
 use-implicit-booleaness-not-len:7:3:7:18::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:HIGH
 use-implicit-booleaness-not-len:11:9:11:34::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:INFERENCE
 use-implicit-booleaness-not-len:14:11:14:22::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:INFERENCE
-comparison-of-constants:39:3:39:28::"Comparison between constants: '0 < 1' has a constant value":HIGH
+comparison-of-constants:39:3:39:28::"Comparison between constants: ""0 < 1"" has a constant value":HIGH
 use-implicit-booleaness-not-len:56:5:56:16::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:INFERENCE
 use-implicit-booleaness-not-len:61:5:61:20::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:HIGH
 use-implicit-booleaness-not-len:64:6:64:17::Do not use `len(SEQUENCE)` without comparison to determine if a sequence is empty:INFERENCE

tests/functional/u/using_constant_test.txt

@@ -24,7 +24,7 @@ using-constant-test:84:36:84:39:test_comprehensions:Using a conditional statemen
 using-constant-test:85:39:85:42:test_comprehensions:Using a conditional statement with a constant value:INFERENCE
 using-constant-test:89:3:89:15::Using a conditional statement with a constant value:INFERENCE
 using-constant-test:93:3:93:18::Using a conditional statement with a constant value:INFERENCE
-comparison-of-constants:117:3:117:8::"Comparison between constants: '2 < 3' has a constant value":HIGH
+comparison-of-constants:117:3:117:8::"Comparison between constants: ""2 < 3"" has a constant value":HIGH
 using-constant-test:156:0:157:8::Using a conditional statement with a constant value:INFERENCE
 using-constant-test:168:3:168:4::Using a conditional statement with a constant value:INFERENCE
 using-constant-test:177:0:178:8::Using a conditional statement with a constant value:INFERENCE