diff --git a/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt b/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt index 32be7ca0..045cbf06 100644 --- a/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt +++ b/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt @@ -4,9 +4,16 @@ import org.xmlpull.v1.XmlPullParser import org.xmlpull.v1.XmlPullParserException internal fun XmlPullParser.nextTrimmedText(): String? { + val startDepth = this.depth return try { this.nextText()?.trim() - } catch (_: XmlPullParserException) { + } catch (exception: XmlPullParserException) { + // Only a child tag is recovered here: nextText() stopped on it, one level deeper. An error + // in the text itself (a bare `&`, an HTML entity like `ä`) leaves the depth unchanged + // and has to reach the caller, which parses the feed again with the XML escaped. Recovering + // from it would drop the text before the error and read on past the closing tag. + if (this.depth <= startDepth) throw exception + // Handle malformed HTML content (e.g., mismatched tag cases like ...) or HTML tags in content // When nextText() throws, there are nested tags, so we manually parse and collect all text // Note: nextText() may have consumed some text before throwing, so we start fresh from current position diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt index 02fb6e01..98170728 100644 --- a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt +++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt @@ -1,6 +1,7 @@ package com.prof18.rssparser import com.prof18.rssparser.exception.RssParsingException +import com.prof18.rssparser.internal.HtmlNamedEntities import com.prof18.rssparser.internal.XmlFetcher import com.prof18.rssparser.internal.XmlParser import com.prof18.rssparser.model.RssChannel @@ -59,12 +60,15 @@ public class RssParser internal constructor( /** * Escapes invalid XML entities in the input string and fixes common XML issues. + * - Replaces HTML named entities (`Ä`, ` `) with numeric character references * - Escapes standalone ampersands that aren't part of valid entity references * - Fixes self-closing or unclosed tags that should be properly closed * - Fixes duplicate closing tags with content between them */ internal fun escapeInvalidXmlEntities(xml: String): String { - return xml + // HTML named entities (Ä,  ) are unknown to XML: turn them into numeric + // references first, so the ampersand escaping below doesn't make them literal text + return HtmlNamedEntities.toNumericReferences(xml) // Fix standalone ampersands in URLs and text .replace(STANDALONE_AMPERSAND_REGEX, "&") // Fix duplicate closing tags with content between them diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt new file mode 100644 index 00000000..e4e2de54 --- /dev/null +++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt @@ -0,0 +1,81 @@ +package com.prof18.rssparser.internal + +/** + * The HTML 4 named character references, which XML doesn't know about. Feeds often copy them + * straight from their HTML (`Ä`, ` `, `—`), and XML parsers reject them. + */ +internal object HtmlNamedEntities { + + private val NAMED_ENTITY_REGEX = Regex("&([A-Za-z][A-Za-z0-9]{1,7});") + + /** + * Replaces every known HTML named entity with the numeric character reference XML accepts, + * so `Ä` becomes `Ä`. The five XML entities and unknown names are left untouched. + */ + fun toNumericReferences(xml: String): String = + NAMED_ENTITY_REGEX.replace(xml) { match -> + val codePoint = CODE_POINTS[match.groupValues[1]] + if (codePoint == null) match.value else "&#$codePoint;" + } + + // Latin-1, U+00A0 to U+00FF, in code point order. + private val LATIN_1 = listOf( + "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect", + "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr", + "deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot", + "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest", + "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil", + "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml", + "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times", + "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig", + "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil", + "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml", + "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide", + "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml", + ) + + // Greek capitals start at U+0391 and lowercase at U+03B1; U+03A2 has no capital. + private val GREEK_UPPER = listOf( + "Alpha", "Beta", "Gamma", "Delta", "Epsilon", "Zeta", "Eta", "Theta", "Iota", + "Kappa", "Lambda", "Mu", "Nu", "Xi", "Omicron", "Pi", "Rho", null, "Sigma", + "Tau", "Upsilon", "Phi", "Chi", "Psi", "Omega", + ) + private val GREEK_LOWER = listOf( + "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", + "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", "sigmaf", "sigma", + "tau", "upsilon", "phi", "chi", "psi", "omega", + ) + + private val OTHERS = mapOf( + "OElig" to 338, "oelig" to 339, "Scaron" to 352, "scaron" to 353, "Yuml" to 376, + "fnof" to 402, "circ" to 710, "tilde" to 732, + "thetasym" to 977, "upsih" to 978, "piv" to 982, + "ensp" to 8194, "emsp" to 8195, "thinsp" to 8201, "zwnj" to 8204, "zwj" to 8205, + "lrm" to 8206, "rlm" to 8207, "ndash" to 8211, "mdash" to 8212, "lsquo" to 8216, + "rsquo" to 8217, "sbquo" to 8218, "ldquo" to 8220, "rdquo" to 8221, "bdquo" to 8222, + "dagger" to 8224, "Dagger" to 8225, "bull" to 8226, "hellip" to 8230, + "permil" to 8240, "prime" to 8242, "Prime" to 8243, "lsaquo" to 8249, + "rsaquo" to 8250, "oline" to 8254, "frasl" to 8260, "euro" to 8364, + "image" to 8465, "weierp" to 8472, "real" to 8476, "trade" to 8482, "alefsym" to 8501, + "larr" to 8592, "uarr" to 8593, "rarr" to 8594, "darr" to 8595, "harr" to 8596, + "crarr" to 8629, "lArr" to 8656, "uArr" to 8657, "rArr" to 8658, "dArr" to 8659, + "hArr" to 8660, "forall" to 8704, "part" to 8706, "exist" to 8707, "empty" to 8709, + "nabla" to 8711, "isin" to 8712, "notin" to 8713, "ni" to 8715, "prod" to 8719, + "sum" to 8721, "minus" to 8722, "lowast" to 8727, "radic" to 8730, "prop" to 8733, + "infin" to 8734, "ang" to 8736, "and" to 8743, "or" to 8744, "cap" to 8745, + "cup" to 8746, "int" to 8747, "there4" to 8756, "sim" to 8764, "cong" to 8773, + "asymp" to 8776, "ne" to 8800, "equiv" to 8801, "le" to 8804, "ge" to 8805, + "sub" to 8834, "sup" to 8835, "nsub" to 8836, "sube" to 8838, "supe" to 8839, + "oplus" to 8853, "otimes" to 8855, "perp" to 8869, "sdot" to 8901, + "lceil" to 8968, "rceil" to 8969, "lfloor" to 8970, "rfloor" to 8971, + "lang" to 9001, "rang" to 9002, "loz" to 9674, + "spades" to 9824, "clubs" to 9827, "hearts" to 9829, "diams" to 9830, + ) + + private val CODE_POINTS: Map = buildMap { + LATIN_1.forEachIndexed { index, name -> put(name, 0xA0 + index) } + GREEK_UPPER.forEachIndexed { index, name -> if (name != null) put(name, 0x391 + index) } + GREEK_LOWER.forEachIndexed { index, name -> put(name, 0x3B1 + index) } + putAll(OTHERS) + } +} diff --git a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt index 6f804e46..ae182ae2 100644 --- a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt +++ b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt @@ -50,6 +50,73 @@ class EscapeInvalidXmlEntitiesTest : XmlParserTestExecutor() { assertEquals(xml, rssParser().escapeInvalidXmlEntities(xml)) } + @Test + fun htmlNamedEntitiesAreTurnedIntoNumericReferences() = runTest { + val xml = "Myrskylä ÄÖ café — 5€ …" + + val escaped = rssParser().escapeInvalidXmlEntities(xml) + + assertEquals( + "Myrskylä ÄÖ café — 5€ …", + escaped, + ) + } + + @Test + fun unknownNamedEntitiesAreEscapedAsText() = runTest { + val xml = "¬anentity; &AUML;" + + val escaped = rssParser().escapeInvalidXmlEntities(xml) + + assertEquals("&notanentity; &AUML;", escaped) + } + + @Test + fun aFeedWithHtmlNamedEntitiesIsParsedWithTheRightCharacters() = runTest { + val xml = """ + + + + Myrskylä + https://myrskyla.fi + Kunta + + Äänestys – tulokset + https://myrskyla.fi/a?x=1&y=2 + Café & ravintola + + + + """.trimIndent() + + val channel = rssParser().parse(xml) + + assertEquals("Myrskylä", channel.title) + val item = channel.items.single() + assertEquals("Äänestys – tulokset", item.title) + assertEquals("https://myrskyla.fi/a?x=1&y=2", item.link) + assertEquals("Café & ravintola", item.description) + } + + @Test + fun aFeedWithABareAmpersandIsParsedOnEveryPlatform() = runTest { + val xml = """ + + + + Tom & Jerry + First + Second + + + """.trimIndent() + + val channel = rssParser().parse(xml) + + assertEquals("Tom & Jerry", channel.title) + assertEquals(listOf("First", "Second"), channel.items.map { it.title }) + } + private fun rssParser(): RssParser = RssParser( xmlFetcher = object : XmlFetcher { override suspend fun fetchXml(url: String): ParserInput = diff --git a/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt b/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt index 84c832a7..33938899 100644 --- a/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt +++ b/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt @@ -13,7 +13,9 @@ import kotlinx.serialization.decodeFromString import kotlinx.serialization.modules.SerializersModule import kotlinx.serialization.modules.polymorphic import kotlinx.serialization.modules.subclass +import nl.adaptivity.xmlutil.EventType import nl.adaptivity.xmlutil.serialization.XML +import nl.adaptivity.xmlutil.xmlStreaming internal class WebXmlParser( private val dispatcher: CoroutineDispatcher, @@ -38,7 +40,7 @@ internal class WebXmlParser( override suspend fun parseXML(input: ParserInput): RssChannel = withContext(dispatcher) { val feedEntity = try { - xml.decodeFromString(input.data) + decodeFeed(input.data) } catch (_: Throwable) { // Fallback to handle XHTML/HTML content and mismatched HTML tags in content/description tags val cleanedData = input.data @@ -62,7 +64,7 @@ internal class WebXmlParser( "$cleanedContent" } try { - xml.decodeFromString(cleanedData) + decodeFeed(cleanedData) } catch (exception: Exception) { throw RssParsingException( message = "Something went wrong when parsing the feed. Please check if the XML is valid", @@ -73,6 +75,37 @@ internal class WebXmlParser( return@withContext feedEntity.toRssChannel(input.baseUrl) } + /** + * The browser's DOMParser doesn't throw on malformed XML (a bare `&`, an HTML entity like + * `ä`): it stops at the error and puts a `parsererror` element in the document. Decoding + * that document gives an empty or truncated feed, so the error is turned into an exception here, + * which lets the caller retry with the escaped XML like on the other platforms. + */ + private fun decodeFeed(data: String): FeedEntity { + if (hasParserError(data)) { + throw IllegalArgumentException("The XML is not well formed") + } + return xml.decodeFromString(data) + } + + private fun hasParserError(data: String): Boolean { + val reader = xmlStreaming.newReader(data) + try { + while (reader.hasNext()) { + if (reader.next() == EventType.START_ELEMENT && reader.localName == PARSER_ERROR_TAG) { + return true + } + } + return false + } finally { + reader.close() + } + } + + private companion object { + private const val PARSER_ERROR_TAG = "parsererror" + } + override fun generateParserInputFromString(rawRssFeed: String): ParserInput = ParserInput( data = rawRssFeed,