Fix AstroidError in similarity checker with imports/signatures ignored

ChangeLog

@@ -192,6 +192,11 @@ Release date: TBA
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ``duplicate-code`` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
 
 What's New in Pylint 2.13.5?
 ============================

doc/whatsnew/2.13.rst

@@ -205,6 +205,11 @@ Other Changes
   Closes #5815
   Closes #5406
 
+* Fixed an ``AstroidError`` in 2.13.0 raised by the ```duplicate-code``` checker with
+  ``ignore-imports`` or ``ignore-signatures`` enabled.
+
+  Closes #6301
+
 * Use the ``tomli`` package instead of ``toml`` to parse ``.toml`` files.
 
  Closes #5885

pylint/checkers/similar.py

@@ -35,10 +35,12 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Dict,
     List,
     NamedTuple,
     NewType,
+    Optional,
     TextIO,
     Tuple,
     Union,
@@ -366,28 +368,20 @@ def append_stream(
             readlines = decoding_stream(stream, encoding).readlines
         else:
             readlines = stream.readlines  # type: ignore[assignment] # hint parameter is incorrectly typed as non-optional
-        try:
-            active_lines: list[str] = []
-            if hasattr(self, "linter"):
-                # Remove those lines that should be ignored because of disables
-                for index, line in enumerate(readlines()):
-                    if self.linter._is_one_message_enabled("R0801", index + 1):  # type: ignore[attr-defined]
-                        active_lines.append(line)
-            else:
-                active_lines = readlines()
-
-            self.linesets.append(
-                LineSet(
-                    streamid,
-                    active_lines,
-                    self.namespace.ignore_comments,
-                    self.namespace.ignore_docstrings,
-                    self.namespace.ignore_imports,
-                    self.namespace.ignore_signatures,
-                )
+
+        self.linesets.append(
+            LineSet(
+                streamid,
+                readlines(),
+                self.namespace.ignore_comments,
+                self.namespace.ignore_docstrings,
+                self.namespace.ignore_imports,
+                self.namespace.ignore_signatures,
+                line_enabled_callback=self.linter._is_one_message_enabled  # type: ignore[attr-defined]
+                if hasattr(self, "linter")
+                else None,
             )
-        except UnicodeDecodeError:
-            pass
+        )
 
     def run(self) -> None:
         """Start looking for similarities and display results on stdout."""
@@ -563,6 +557,7 @@ def stripped_lines(
     ignore_docstrings: bool,
     ignore_imports: bool,
     ignore_signatures: bool,
+    line_enabled_callback: Callable | None = None,
 ) -> list[LineSpecifs]:
     """Return tuples of line/line number/line type with leading/trailing whitespace and any ignored code features removed.
 
@@ -571,6 +566,7 @@ def stripped_lines(
     :param ignore_docstrings: if true, any line that is a docstring is removed from the result
     :param ignore_imports: if true, any line that is an import is removed from the result
     :param ignore_signatures: if true, any line that is part of a function signature is removed from the result
+    :param line_enabled_callback: If called with "R0801" and a line number, a return value of False will disregard the line
     :return: the collection of line/line number/line type tuples
     """
     if ignore_imports or ignore_signatures:
@@ -622,6 +618,10 @@ def _get_functions(
     strippedlines = []
     docstring = None
     for lineno, line in enumerate(lines, start=1):
+        if line_enabled_callback is not None and not line_enabled_callback(
+            "R0801", lineno
+        ):
+            continue
         line = line.strip()
         if ignore_docstrings:
             if not docstring:
@@ -668,12 +668,21 @@ def __init__(
         ignore_docstrings: bool = False,
         ignore_imports: bool = False,
         ignore_signatures: bool = False,
+        line_enabled_callback: Callable | None = None,
     ) -> None:
         self.name = name
         self._real_lines = lines
-        self._stripped_lines = stripped_lines(
-            lines, ignore_comments, ignore_docstrings, ignore_imports, ignore_signatures
-        )
+        try:
+            self._stripped_lines = stripped_lines(
+                lines,
+                ignore_comments,
+                ignore_docstrings,
+                ignore_imports,
+                ignore_signatures,
+                line_enabled_callback=line_enabled_callback,
+            )
+        except UnicodeDecodeError:
+            self._stripped_lines = []
 
     def __str__(self):
         return f"<Lineset for {self.name}>"

tests/regrtest_data/duplicate_code/raw_strings_disable_line_some/first.py

@@ -0,0 +1,12 @@
+r"""A raw docstring.
+"""
+
+
+def look_busy():
+    xxxx = 1  # pylint: disable=duplicate-code
+    yyyy = 2  # pylint: disable=duplicate-code
+    zzzz = 3
+    wwww = 4
+    vvvv = xxxx + yyyy + zzzz + wwww
+    vvvv *= 2
+    return vvvv

tests/regrtest_data/duplicate_code/raw_strings_disable_line_some/second.py

@@ -0,0 +1,12 @@
+r"""A raw docstring.
+"""
+
+
+def look_busy():
+    xxxx = 1
+    yyyy = 2
+    zzzz = 3
+    wwww = 4
+    vvvv = xxxx + yyyy + zzzz + wwww
+    vvvv *= 2
+    return vvvv

tests/test_similar.py

@@ -35,7 +35,13 @@ def _runtest(self, args: list[str], code: int) -> None:
     @staticmethod
     def _run_pylint(args: list[str], out: TextIO) -> int:
         """Runs pylint with a patched output."""
-        args = args + ["--persistent=no"]
+        args = args + [
+            "--persistent=no",
+            "--enable=astroid-error",
+            # Enable functionality that will build another ast
+            "ignore-imports=y",
+            "ignore-signatures=y",
+        ]
         with _patch_streams(out):
             with pytest.raises(SystemExit) as cm:
                 with warnings.catch_warnings():
@@ -54,8 +60,10 @@ def _test_output(self, args: list[str], expected_output: str) -> None:
         out = StringIO()
         self._run_pylint(args, out=out)
         actual_output = self._clean_paths(out.getvalue())
+        actual_output_stripped = actual_output.strip()
         expected_output = self._clean_paths(expected_output)
-        assert expected_output.strip() in actual_output.strip()
+        assert expected_output.strip() in actual_output_stripped
+        assert "Fatal error" not in actual_output_stripped
 
     def test_duplicate_code_raw_strings_all(self) -> None:
         """Test similar lines in 3 similar files."""
@@ -108,6 +116,17 @@ def test_duplicate_code_raw_strings_disable_line_end(self) -> None:
             expected_output=expected_output,
         )
 
+    def test_duplicate_code_raw_strings_disable_line_some(self) -> None:
+        """Tests disabling duplicate-code in the first two lines but not all lines
+        of a piece of similar code.
+        """
+        path = join(DATA, "raw_strings_disable_line_some")
+        expected_output = "Similar lines in 2 files"
+        self._test_output(
+            [path, "--disable=all", "--enable=duplicate-code"],
+            expected_output=expected_output,
+        )
+
     def test_duplicate_code_raw_strings_disable_scope(self) -> None:
         """Tests disabling duplicate-code at an inner scope level."""
         path = join(DATA, "raw_strings_disable_scope")