Fix line separator issues (#1887)

* Fix line separator issues * Test fix * Handle just CR as well, add better test * Add escaped \r\n to tests
Kotlin · May 6, 2021 · 326048c · 326048c
1 parent 6337858
commit 326048c
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 23 deletions.
diff --git a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
@@ -12,8 +12,24 @@ import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor
 import org.intellij.markdown.flavours.gfm.GFMTokenTypes
 import org.jetbrains.dokka.base.parsers.factories.DocTagsFromIElementFactory
 import org.jetbrains.dokka.links.DRI
-import org.jetbrains.dokka.model.doc.*
+import org.jetbrains.dokka.model.doc.Author
+import org.jetbrains.dokka.model.doc.Constructor
+import org.jetbrains.dokka.model.doc.CustomTagWrapper
+import org.jetbrains.dokka.model.doc.Description
+import org.jetbrains.dokka.model.doc.DocTag
+import org.jetbrains.dokka.model.doc.DocumentationLink
+import org.jetbrains.dokka.model.doc.DocumentationNode
+import org.jetbrains.dokka.model.doc.Param
+import org.jetbrains.dokka.model.doc.Property
+import org.jetbrains.dokka.model.doc.Receiver
+import org.jetbrains.dokka.model.doc.Return
+import org.jetbrains.dokka.model.doc.Sample
+import org.jetbrains.dokka.model.doc.See
+import org.jetbrains.dokka.model.doc.Since
 import org.jetbrains.dokka.model.doc.Suppress
+import org.jetbrains.dokka.model.doc.TagWrapper
+import org.jetbrains.dokka.model.doc.Text
+import org.jetbrains.dokka.model.doc.Throws
 import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag
 import org.jetbrains.kotlin.kdoc.psi.impl.KDocSection
 import org.jetbrains.kotlin.kdoc.psi.impl.KDocTag
@@ -23,7 +39,7 @@ import org.intellij.markdown.parser.MarkdownParser as IntellijMarkdownParser
 
 open class MarkdownParser(
     private val externalDri: (String) -> DRI?,
-    private val kdocLocation: String?
+    private val kdocLocation: String?,
 ) : Parser() {
 
     private lateinit var destinationLinksMap: Map<String, String>
@@ -37,7 +53,7 @@ open class MarkdownParser(
         return visitNode(markdownAstRoot)
     }
 
-    override fun preparse(text: String) = text
+    override fun preparse(text: String) = text.replace("\r\n", "\n").replace("\r", "\n")
 
     override fun parseTagWithBody(tagName: String, content: String): TagWrapper =
         when (tagName) {
@@ -347,27 +363,32 @@ open class MarkdownParser(
             MarkdownElementTypes.ATX_3,
             MarkdownElementTypes.ATX_4,
             MarkdownElementTypes.ATX_5,
-            MarkdownElementTypes.ATX_6 -> headersHandler(node)
+            MarkdownElementTypes.ATX_6,
+            -> headersHandler(node)
             MarkdownTokenTypes.HORIZONTAL_RULE -> horizontalRulesHandler(node)
             MarkdownElementTypes.STRONG -> strongHandler(node)
             MarkdownElementTypes.EMPH -> emphasisHandler(node)
             MarkdownElementTypes.FULL_REFERENCE_LINK,
-            MarkdownElementTypes.SHORT_REFERENCE_LINK -> referenceLinksHandler(node)
+            MarkdownElementTypes.SHORT_REFERENCE_LINK,
+            -> referenceLinksHandler(node)
             MarkdownElementTypes.INLINE_LINK -> inlineLinksHandler(node)
             MarkdownElementTypes.AUTOLINK -> autoLinksHandler(node)
             MarkdownElementTypes.BLOCK_QUOTE -> blockquotesHandler(node)
             MarkdownElementTypes.UNORDERED_LIST,
-            MarkdownElementTypes.ORDERED_LIST -> listsHandler(node)
+            MarkdownElementTypes.ORDERED_LIST,
+            -> listsHandler(node)
             MarkdownElementTypes.CODE_BLOCK -> codeBlocksHandler(node)
             MarkdownElementTypes.CODE_FENCE -> codeFencesHandler(node)
             MarkdownElementTypes.CODE_SPAN -> codeSpansHandler(node)
             MarkdownElementTypes.IMAGE -> imagesHandler(node)
             MarkdownElementTypes.HTML_BLOCK,
             MarkdownTokenTypes.HTML_TAG,
-            MarkdownTokenTypes.HTML_BLOCK_CONTENT -> rawHtmlHandler(node)
+            MarkdownTokenTypes.HTML_BLOCK_CONTENT,
+            -> rawHtmlHandler(node)
             MarkdownTokenTypes.HARD_LINE_BREAK -> DocTagsFromIElementFactory.getInstance(node.type)
             MarkdownTokenTypes.CODE_FENCE_CONTENT,
-            MarkdownTokenTypes.CODE_LINE -> codeLineHandler(node)
+            MarkdownTokenTypes.CODE_LINE,
+            -> codeLineHandler(node)
             MarkdownTokenTypes.TEXT -> textHandler(node)
             MarkdownElementTypes.MARKDOWN_FILE -> markdownFileHandler(node)
             GFMElementTypes.STRIKETHROUGH -> strikeThroughHandler(node)
@@ -482,7 +503,7 @@ open class MarkdownParser(
         fun parseFromKDocTag(
             kDocTag: KDocTag?,
             externalDri: (String) -> DRI?,
-            kdocLocation: String?
+            kdocLocation: String?,
         ): DocumentationNode {
             return if (kDocTag == null) {
                 DocumentationNode(emptyList())

diff --git a/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt b/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt
@@ -1,8 +1,12 @@
 package parsers
 
-import org.jetbrains.dokka.base.parsers.moduleAndPackage.*
+import org.jetbrains.dokka.base.parsers.moduleAndPackage.IllegalModuleAndPackageDocumentation
 import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentation.Classifier.Module
 import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentation.Classifier.Package
+import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationFile
+import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationFragment
+import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationSource
+import org.jetbrains.dokka.base.parsers.moduleAndPackage.parseModuleAndPackageDocumentationFragments
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
@@ -12,9 +16,7 @@ import java.nio.file.Path
 
 class ParseModuleAndPackageDocumentationFragmentsTest {
 
-    @Test
-    fun `basic example`() {
-
+    private fun testBasicExample(lineSeperator: String = "\n") {
         val source = source(
             """
                 # Module kotlin-demo
@@ -23,11 +25,11 @@ class ParseModuleAndPackageDocumentationFragmentsTest {
                 # Package org.jetbrains.kotlin.demo
                 Package demo description
                 ## Level 2 heading
-                Heading 2
+                Heading 2\r\n
         
                 # Package org.jetbrains.kotlin.demo2
                 Package demo2 description
-                """.trimIndent()
+                """.trimIndent().replace("\n", lineSeperator)
         )
         val fragments = parseModuleAndPackageDocumentationFragments(source)
 
@@ -42,7 +44,7 @@ class ParseModuleAndPackageDocumentationFragmentsTest {
                 ModuleAndPackageDocumentationFragment(
                     classifier = Package,
                     name = "org.jetbrains.kotlin.demo",
-                    documentation = "Package demo description\n## Level 2 heading\nHeading 2",
+                    documentation = "Package demo description${lineSeperator}## Level 2 heading${lineSeperator}Heading 2\\r\\n",
                     source = source
                 ),
                 ModuleAndPackageDocumentationFragment(
@@ -56,6 +58,16 @@ class ParseModuleAndPackageDocumentationFragmentsTest {
         )
     }
 
+    @Test
+    fun `basic example`() {
+        testBasicExample()
+    }
+
+    @Test
+    fun `CRLF line seperators`() {
+        testBasicExample("\r\n")
+    }
+
     @Test
     fun `no module name specified fails`() {
         val exception = assertThrows<IllegalModuleAndPackageDocumentation> {

diff --git a/plugins/base/src/test/kotlin/transformers/ContextModuleAndPackageDocumentationReaderTest1.kt b/plugins/base/src/test/kotlin/transformers/ContextModuleAndPackageDocumentationReaderTest1.kt
@@ -14,7 +14,6 @@ import org.junit.jupiter.api.assertThrows
 import testApi.testRunner.TestDokkaConfigurationBuilder
 import testApi.testRunner.dModule
 import testApi.testRunner.dPackage
-import java.lang.IllegalStateException
 
 class ContextModuleAndPackageDocumentationReaderTest1 : AbstractContextModuleAndPackageDocumentationReaderTest() {
 
@@ -30,11 +29,11 @@ class ContextModuleAndPackageDocumentationReaderTest1 : AbstractContextModuleAnd
             This is moduleA
             
             # Package sample.a
-            This is package sample.a
+            This is package sample.a\r\n
             
             # Package noise.b
             This will just add some noise
-            """.trimIndent()
+            """.trimIndent().replace("\n", "\r\n")
         )
 
         includeSourceSetB.writeText(
@@ -114,9 +113,11 @@ class ContextModuleAndPackageDocumentationReaderTest1 : AbstractContextModuleAnd
     fun `assert moduleA with unknown source set`() {
         assertThrows<IllegalStateException>(
             "Expected no documentation received for module with unknown sourceSet"
-        ) { reader[
-                dModule("moduleA", sourceSets = setOf(configurationBuilder.unattachedSourceSet { name = "unknown" }))
-        ]}
+        ) {
+            reader[
+                    dModule("moduleA", sourceSets = setOf(configurationBuilder.unattachedSourceSet { name = "unknown" }))
+            ]
+        }
     }
 
     @Test
@@ -140,7 +141,7 @@ class ContextModuleAndPackageDocumentationReaderTest1 : AbstractContextModuleAnd
         val documentation = reader[dPackage(DRI("sample.a"), sourceSets = setOf(sourceSetA))]
         assertEquals(1, documentation.keys.size, "Expected only one entry from sourceSetA")
         assertEquals(sourceSetA, documentation.keys.single(), "Expected only one entry from sourceSetA")
-        assertEquals("This is package sample.a", documentation.texts.single())
+        assertEquals("This is package sample.a\\r\\n", documentation.texts.single())
     }
 
     @Test