diff --git a/isort/output.py b/isort/output.py
index 1d51171d3..017ae4d4e 100644
--- a/isort/output.py
+++ b/isort/output.py
@@ -543,25 +543,25 @@ def _with_straight_imports(
         import_definition = []
         if module in parsed.as_map["straight"]:
             if parsed.imports[section]["straight"][module]:
-                import_definition.append(f"{import_type} {module}")
+                import_definition.append((f"{import_type} {module}", module))
             import_definition.extend(
-                f"{import_type} {module} as {as_import}"
+                (f"{import_type} {module} as {as_import}", f"{module} as {as_import}")
                 for as_import in parsed.as_map["straight"][module]
             )
         else:
-            import_definition.append(f"{import_type} {module}")
+            import_definition.append((f"{import_type} {module}", module))
 
         comments_above = parsed.categorized_comments["above"]["straight"].pop(module, None)
         if comments_above:
             output.extend(comments_above)
         output.extend(
             with_comments(
-                parsed.categorized_comments["straight"].get(module),
+                parsed.categorized_comments["straight"].get(imodule),
                 idef,
                 removed=config.ignore_comments,
                 comment_prefix=config.comment_prefix,
             )
-            for idef in import_definition
+            for idef, imodule in import_definition
         )
 
     return output
diff --git a/isort/parse.py b/isort/parse.py
index d93f559e2..307015e74 100644
--- a/isort/parse.py
+++ b/isort/parse.py
@@ -410,9 +410,16 @@ def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedConte
                                 f"{top_level_module}.__combined_as__", []
                             )
                         else:
-                            attach_comments_to = categorized_comments["straight"].setdefault(
-                                module, []
-                            )
+                            if type_of_import == "from" or (
+                                config.remove_redundant_aliases and as_name == module.split(".")[-1]
+                            ):
+                                attach_comments_to = categorized_comments["straight"].setdefault(
+                                    module, []
+                                )
+                            else:
+                                attach_comments_to = categorized_comments["straight"].setdefault(
+                                    f"{module} as {as_name}", []
+                                )
                     del just_imports[as_index : as_index + 2]
 
             if type_of_import == "from":
diff --git a/tests/integration/test_projects_using_isort.py b/tests/integration/test_projects_using_isort.py
index 2258cbfae..17cea29c4 100644
--- a/tests/integration/test_projects_using_isort.py
+++ b/tests/integration/test_projects_using_isort.py
@@ -61,7 +61,19 @@ def test_habitat_lab(tmpdir):
 
 def test_tmuxp(tmpdir):
     git_clone("https://github.com/tmux-python/tmuxp.git", tmpdir)
-    run_isort([str(tmpdir), "--skip", "cli.py", "--skip", "test_workspacebuilder.py"])
+    run_isort(
+        [
+            str(tmpdir),
+            "--skip",
+            "cli.py",
+            "--skip",
+            "test_workspacebuilder.py",
+            "--skip",
+            "test_cli.py",
+            "--skip",
+            "workspacebuilder.py",
+        ]
+    )
 
 
 def test_websockets(tmpdir):
diff --git a/tests/unit/test_regressions.py b/tests/unit/test_regressions.py
index 89fa09274..7ba1da770 100644
--- a/tests/unit/test_regressions.py
+++ b/tests/unit/test_regressions.py
@@ -1515,3 +1515,25 @@ def test_isort_adding_second_comma_issue_1621():
 )
 """
     )
+
+
+def test_isort_shouldnt_duplicate_comments_issue_1631():
+    assert isort.check_code(
+        """
+import a  # a comment
+import a as b  # b comment
+""",
+        show_diff=True,
+    )
+    assert (
+        isort.code(
+            """
+import a  # a comment
+import a as a  # b comment
+""",
+            remove_redundant_aliases=True,
+        )
+        == """
+import a  # a comment; b comment
+"""
+    )