diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
index 3856554c..2cf6d56e 100644
--- a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
+++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
@@ -63,41 +63,30 @@ public class RssParser internal constructor(
* - Escapes standalone ampersands that aren't part of valid entity references
* - Fixes self-closing or unclosed tags that should be properly closed
* - Fixes duplicate closing tags with content between them
- * - Handles special cases for ampersands in URLs and text content
*/
- private fun escapeInvalidXmlEntities(xml: String): String {
+ internal fun escapeInvalidXmlEntities(xml: String): String {
return xml
// Fix standalone ampersands in URLs and text
- .replace(
- Regex("&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))"),
- "&"
- )
+ .replace(STANDALONE_AMPERSAND_REGEX, "&")
// Fix duplicate closing tags with content between them
// Example: ->
- .replace(
- Regex("<([^>]+)>\\1>([^<]+|)\\1>"),
- "<$1>$2$1>"
- )
+ .replace(DUPLICATE_CLOSING_TAG_REGEX, "<$1>$2$1>")
// Fix self-closing tags, but only if they don't already have content
// This regex checks that there's no content between the opening and closing tags
- .replace(
- Regex("<(link|source|category|guid|enclosure|media:content|media:thumbnail)([^>]*?)>\\s*\\1>"),
- "<$1$2>$1>"
- )
+ .replace(EMPTY_FEED_TAG_REGEX, "<$1$2>$1>")
// Fix other common HTML tags that might be self-closing
- .replace(
- Regex("<(meta|img|br|hr|input|area|base|col|embed|keygen|param|track|wbr)([^>]*?)/?>(?!\\1>)"),
- "<$1$2>$1>"
- )
- // Additional pass to catch any ampersands in CDATA sections or attribute values that might have been missed
- .replace(
- Regex("()"),
- "$1&$3"
- )
- .replace(
- Regex("=\"(.*?)&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))(.*?)\""),
- "=\"$1&$3\""
- )
+ .replace(VOID_HTML_TAG_REGEX, "<$1$2>$1>")
+ }
+
+ private companion object {
+ private val STANDALONE_AMPERSAND_REGEX =
+ Regex("&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))")
+ private val DUPLICATE_CLOSING_TAG_REGEX =
+ Regex("<([^>]+)>\\1>([^<]+|)\\1>")
+ private val EMPTY_FEED_TAG_REGEX =
+ Regex("<(link|source|category|guid|enclosure|media:content|media:thumbnail)([^>]*?)>\\s*\\1>")
+ private val VOID_HTML_TAG_REGEX =
+ Regex("<(meta|img|br|hr|input|area|base|col|embed|keygen|param|track|wbr)([^>]*?)/?>(?!\\1>)")
}
}
diff --git a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt
new file mode 100644
index 00000000..6f804e46
--- /dev/null
+++ b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt
@@ -0,0 +1,87 @@
+package com.prof18.rssparser
+
+import com.prof18.rssparser.internal.ParserInput
+import com.prof18.rssparser.internal.XmlFetcher
+import kotlinx.coroutines.test.runTest
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class EscapeInvalidXmlEntitiesTest : XmlParserTestExecutor() {
+
+ /**
+ * A feed URL pointing at a website hands the escaping fallback a full HTML page, which is one
+ * very long line. The test passes or it times out, there is nothing in between.
+ */
+ @Test
+ fun escapingALargeMinifiedHtmlPageTerminates() = runTest {
+ val html = buildMinifiedHtmlPage()
+ assertTrue(html.length > 800_000, "expected a page big enough to be representative")
+ assertEquals(0, html.count { it == '\n' }, "the page has to be one single line")
+
+ val escaped = rssParser().escapeInvalidXmlEntities(html)
+
+ assertFalse(BARE_AMPERSAND_REGEX.containsMatchIn(escaped))
+ }
+
+ @Test
+ fun bareAmpersandsAreEscapedEverywhereIncludingCdataAndAttributes() = runTest {
+ val xml = """
+
+ https://example.com/a?ref=home&pos=1
+
+
+
+ """.trimIndent()
+
+ val escaped = rssParser().escapeInvalidXmlEntities(xml)
+
+ assertTrue(escaped.contains("?ref=home&pos=1"))
+ assertTrue(escaped.contains("?token=x&exp=2"))
+ assertTrue(escaped.contains("Tom & Jerry, R&D and Q&A"))
+ assertFalse(BARE_AMPERSAND_REGEX.containsMatchIn(escaped))
+ }
+
+ @Test
+ fun alreadyValidEntitiesAreLeftAlone() = runTest {
+ val xml = "
A & B <C> "D" 'E' & &"
+
+ assertEquals(xml, rssParser().escapeInvalidXmlEntities(xml))
+ }
+
+ private fun rssParser(): RssParser = RssParser(
+ xmlFetcher = object : XmlFetcher {
+ override suspend fun fetchXml(url: String): ParserInput =
+ error("not used by these tests")
+
+ override suspend fun fetchXmlAsString(url: String): String =
+ error("not used by these tests")
+ },
+ xmlParser = createXmlParser()
+ )
+
+ /**
+ * Roughly what a news homepage looks like to the parser: no newlines, and thousands of
+ * `attribute="…"` pairs holding query strings with bare ampersands in them.
+ */
+ private fun buildMinifiedHtmlPage(): String = buildString {
+ append("")
+ append("Nachrichten")
+ repeat(HTML_ELEMENT_COUNT) { index ->
+ append("