diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt index 3856554c..2cf6d56e 100644 --- a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt +++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt @@ -63,41 +63,30 @@ public class RssParser internal constructor( * - Escapes standalone ampersands that aren't part of valid entity references * - Fixes self-closing or unclosed tags that should be properly closed * - Fixes duplicate closing tags with content between them - * - Handles special cases for ampersands in URLs and text content */ - private fun escapeInvalidXmlEntities(xml: String): String { + internal fun escapeInvalidXmlEntities(xml: String): String { return xml // Fix standalone ampersands in URLs and text - .replace( - Regex("&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))"), - "&" - ) + .replace(STANDALONE_AMPERSAND_REGEX, "&") // Fix duplicate closing tags with content between them // Example: -> - .replace( - Regex("<([^>]+)>([^<]+|)"), - "<$1>$2" - ) + .replace(DUPLICATE_CLOSING_TAG_REGEX, "<$1>$2") // Fix self-closing tags, but only if they don't already have content // This regex checks that there's no content between the opening and closing tags - .replace( - Regex("<(link|source|category|guid|enclosure|media:content|media:thumbnail)([^>]*?)>\\s*"), - "<$1$2>" - ) + .replace(EMPTY_FEED_TAG_REGEX, "<$1$2>") // Fix other common HTML tags that might be self-closing - .replace( - Regex("<(meta|img|br|hr|input|area|base|col|embed|keygen|param|track|wbr)([^>]*?)/?>(?!)"), - "<$1$2>" - ) - // Additional pass to catch any ampersands in CDATA sections or attribute values that might have been missed - .replace( - Regex("()"), - "$1&$3" - ) - .replace( - Regex("=\"(.*?)&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))(.*?)\""), - "=\"$1&$3\"" - ) + .replace(VOID_HTML_TAG_REGEX, "<$1$2>") + } + + private companion object { + private val STANDALONE_AMPERSAND_REGEX = + Regex("&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))") + private val DUPLICATE_CLOSING_TAG_REGEX = + Regex("<([^>]+)>([^<]+|)") + private val EMPTY_FEED_TAG_REGEX = + Regex("<(link|source|category|guid|enclosure|media:content|media:thumbnail)([^>]*?)>\\s*") + private val VOID_HTML_TAG_REGEX = + Regex("<(meta|img|br|hr|input|area|base|col|embed|keygen|param|track|wbr)([^>]*?)/?>(?!)") } } diff --git a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt new file mode 100644 index 00000000..6f804e46 --- /dev/null +++ b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt @@ -0,0 +1,87 @@ +package com.prof18.rssparser + +import com.prof18.rssparser.internal.ParserInput +import com.prof18.rssparser.internal.XmlFetcher +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class EscapeInvalidXmlEntitiesTest : XmlParserTestExecutor() { + + /** + * A feed URL pointing at a website hands the escaping fallback a full HTML page, which is one + * very long line. The test passes or it times out, there is nothing in between. + */ + @Test + fun escapingALargeMinifiedHtmlPageTerminates() = runTest { + val html = buildMinifiedHtmlPage() + assertTrue(html.length > 800_000, "expected a page big enough to be representative") + assertEquals(0, html.count { it == '\n' }, "the page has to be one single line") + + val escaped = rssParser().escapeInvalidXmlEntities(html) + + assertFalse(BARE_AMPERSAND_REGEX.containsMatchIn(escaped)) + } + + @Test + fun bareAmpersandsAreEscapedEverywhereIncludingCdataAndAttributes() = runTest { + val xml = """ + + https://example.com/a?ref=home&pos=1 + + + + """.trimIndent() + + val escaped = rssParser().escapeInvalidXmlEntities(xml) + + assertTrue(escaped.contains("?ref=home&pos=1")) + assertTrue(escaped.contains("?token=x&exp=2")) + assertTrue(escaped.contains("Tom & Jerry, R&D and Q&A")) + assertFalse(BARE_AMPERSAND_REGEX.containsMatchIn(escaped)) + } + + @Test + fun alreadyValidEntitiesAreLeftAlone() = runTest { + val xml = "A & B <C> "D" 'E' & &" + + assertEquals(xml, rssParser().escapeInvalidXmlEntities(xml)) + } + + private fun rssParser(): RssParser = RssParser( + xmlFetcher = object : XmlFetcher { + override suspend fun fetchXml(url: String): ParserInput = + error("not used by these tests") + + override suspend fun fetchXmlAsString(url: String): String = + error("not used by these tests") + }, + xmlParser = createXmlParser() + ) + + /** + * Roughly what a news homepage looks like to the parser: no newlines, and thousands of + * `attribute="…"` pairs holding query strings with bare ampersands in them. + */ + private fun buildMinifiedHtmlPage(): String = buildString { + append("") + append("Nachrichten") + repeat(HTML_ELEMENT_COUNT) { index -> + append("
") + append("") + append("\"Bild") + append("Schlagzeile $index — mehr dazu") + append("
") + } + append("") + } + + private companion object { + // ~4.500 elements of ~200 chars lands around 900 KB, the size of a real news homepage. + private const val HTML_ELEMENT_COUNT = 4_500 + private val BARE_AMPERSAND_REGEX = + Regex("&(?!(amp;|lt;|gt;|quot;|apos;|#[0-9]+;|#x[0-9a-fA-F]+;))") + } +}